fix(perf): speed up summary retrieval of all time interval (resolve #336)

migrations/20220317_align_num_heartbeats.go

@@ -0,0 +1,56 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220317-align_num_heartbeats"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			logbuch.Info("this may take a while!")
+
+			// find all summaries whose num_heartbeats is zero even though they have items
+			var faultyIds []uint
+
+			if err := db.Model(&models.Summary{}).
+				Distinct("summaries.id").
+				Joins("INNER JOIN summary_items ON summaries.num_heartbeats = 0 AND summaries.id = summary_items.summary_id").
+				Scan(&faultyIds).Error; err != nil {
+				return err
+			}
+
+			// update their heartbeats counter
+			result := db.
+				Table("summaries AS s1").
+				Where("s1.id IN ?", faultyIds).
+				Update(
+					"num_heartbeats",
+					db.
+						Model(&models.Heartbeat{}).
+						Select("COUNT(*)").
+						Where("user_id = ?", gorm.Expr("s1.user_id")).
+						Where("time BETWEEN ? AND ?", gorm.Expr("s1.from_time"), gorm.Expr("s1.to_time")),
+				)
+
+			if err := result.Error; err != nil {
+				return err
+			}
+
+			logbuch.Info("corrected heartbeats counter of %d summaries", result.RowsAffected)
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

models/summary.go

@@ -101,7 +101,23 @@ func (s *Summary) MappedItems() map[uint8]*SummaryItems {
 }
 
 func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
-	return s.MappedItems()[summaryType]
+	switch summaryType {
+	case SummaryProject:
+		return &s.Projects
+	case SummaryLanguage:
+		return &s.Languages
+	case SummaryEditor:
+		return &s.Editors
+	case SummaryOS:
+		return &s.OperatingSystems
+	case SummaryMachine:
+		return &s.Machines
+	case SummaryLabel:
+		return &s.Labels
+	case SummaryBranch:
+		return &s.Branches
+	}
+	return nil
 }
 
 func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {

repositories/summary.go

@@ -18,15 +18,15 @@ func (r *SummaryRepository) GetAll() ([]*models.Summary, error) {
 	var summaries []*models.Summary
 	if err := r.db.
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -44,15 +44,15 @@ func (r *SummaryRepository) GetByUserWithin(user *models.User, from, to time.Tim
 		Where("from_time >= ?", from.Local()).
 		Where("to_time <= ?", to.Local()).
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -74,3 +74,32 @@ func (r *SummaryRepository) DeleteByUser(userId string) error {
 	}
 	return nil
 }
+
+// inplace
+func (r *SummaryRepository) populateItems(summaries []*models.Summary) error {
+	summaryMap := map[uint]*models.Summary{}
+	summaryIds := make([]uint, len(summaries))
+	for i, s := range summaries {
+		if s.NumHeartbeats == 0 {
+			continue
+		}
+		summaryMap[s.ID] = s
+		summaryIds[i] = s.ID
+	}
+
+	var items []*models.SummaryItem
+
+	if err := r.db.
+		Model(&models.SummaryItem{}).
+		Where("summary_id in ?", summaryIds).
+		Find(&items).Error; err != nil {
+		return err
+	}
+
+	for _, item := range items {
+		l := summaryMap[item.SummaryID].ItemsByType(item.Type)
+		*l = append(*l, item)
+	}
+
+	return nil
+}

version.txt

@@ -1 +1 @@
-2.2.6
+2.2.7