feat: add ability to regenerate summaries

fix: database cascade settings
chore: debug log mode for gorm queries is back

main.go

@@ -63,6 +63,9 @@ func main() {
 		db.Raw("PRAGMA foreign_keys = ON;")
 	}
 
+	if config.IsDev() {
+		db = db.Debug()
+	}
 	sqlDb, _ := db.DB()
 	sqlDb.SetMaxIdleConns(int(config.Db.MaxConn))
 	sqlDb.SetMaxOpenConns(int(config.Db.MaxConn))
@@ -103,7 +106,7 @@ func main() {
 	summaryHandler := routes.NewSummaryHandler(summaryService)
 	healthHandler := routes.NewHealthHandler(db)
 	heartbeatHandler := routes.NewHeartbeatHandler(heartbeatService, languageMappingService)
-	settingsHandler := routes.NewSettingsHandler(userService, languageMappingService)
+	settingsHandler := routes.NewSettingsHandler(userService, summaryService, aggregationService, languageMappingService)
 	publicHandler := routes.NewIndexHandler(userService, keyValueService)
 	wakatimeV1AllHandler := wtV1Routes.NewAllTimeHandler(summaryService)
 	wakatimeV1SummariesHandler := wtV1Routes.NewSummariesHandler(summaryService)
@@ -148,10 +151,11 @@ func main() {
 	// Settings Routes
 	settingsRouter.Methods(http.MethodGet).HandlerFunc(settingsHandler.GetIndex)
 	settingsRouter.Path("/credentials").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCredentials)
-	settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCreateLanguageMapping)
+	settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostLanguageMapping)
 	settingsRouter.Path("/language_mappings/delete").Methods(http.MethodPost).HandlerFunc(settingsHandler.DeleteLanguageMapping)
 	settingsRouter.Path("/reset").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostResetApiKey)
 	settingsRouter.Path("/badges").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostToggleBadges)
+	settingsRouter.Path("/regenerate").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostRegenerateSummaries)
 
 	// API Routes
 	apiRouter.Path("/heartbeat").Methods(http.MethodPost).HandlerFunc(heartbeatHandler.ApiPost)

migrations/common/custom_pre.go

@@ -30,6 +30,71 @@ func init() {
 			},
 			name: "rename language mappings table",
 		},
+		{
+			f: func(db *gorm.DB, cfg *config.Config) error {
+				// drop all already existing foreign key constraints
+				// afterwards let them be re-created by auto migrate with the newly introduced cascade settings,
+
+				migrator := db.Migrator()
+				const lookupKey = "20201106-migration_cascade_constraints"
+
+				if cfg.Db.Dialect == "sqlite3" {
+					// https://stackoverflow.com/a/1884893/3112139
+					// unfortunately, we can't migrate existing sqlite databases to the newly introduced cascade settings
+					// things like deleting all summaries won't work in those cases unless an entirely new db is created
+					log.Println("not attempting to drop and regenerate constraints on sqlite")
+					return nil
+				}
+
+				if !migrator.HasTable(&models.KeyStringValue{}) {
+					log.Println("key-value table not yet existing")
+					return nil
+				}
+
+				condition := "key = ?"
+				if cfg.Db.Dialect == "mysql" {
+					condition = "`key` = ?"
+				}
+				lookupResult := db.Where(condition, lookupKey).First(&models.KeyStringValue{})
+				if lookupResult.Error == nil && lookupResult.RowsAffected > 0 {
+					log.Println("no need to migrate cascade constraints")
+					return nil
+				}
+
+				// SELECT * FROM INFORMATION_SCHEMA.table_constraints;
+				constraints := map[string]interface{}{
+					"fk_summaries_editors":           &models.SummaryItem{},
+					"fk_summaries_languages":         &models.SummaryItem{},
+					"fk_summaries_machines":          &models.SummaryItem{},
+					"fk_summaries_operating_systems": &models.SummaryItem{},
+					"fk_summaries_projects":          &models.SummaryItem{},
+					"fk_summary_items_summary":       &models.SummaryItem{},
+					"fk_summaries_user":              &models.Summary{},
+					"fk_language_mappings_user":      &models.LanguageMapping{},
+					"fk_heartbeats_user":             &models.Heartbeat{},
+					"fk_aliases_user":                &models.Alias{},
+				}
+
+				for name, table := range constraints {
+					if migrator.HasConstraint(table, name) {
+						log.Printf("dropping constraint '%s'", name)
+						if err := migrator.DropConstraint(table, name); err != nil {
+							return err
+						}
+					}
+				}
+
+				if err := db.Create(&models.KeyStringValue{
+					Key:   lookupKey,
+					Value: "done",
+				}).Error; err != nil {
+					return err
+				}
+
+				return nil
+			},
+			name: "add cascade constraints",
+		},
 	}
 }
 

models/alias.go

@@ -2,7 +2,7 @@ package models
 
 type Alias struct {
 	ID     uint   `gorm:"primary_key"`
-	Type   uint8  `gorm:"not null; index:idx_alias_type_key"`
+	Type   uint8  `gorm:"not null; index:idx_alias_type_key; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
 	User   *User  `json:"-" gorm:"not null"`
 	UserID string `gorm:"not null; index:idx_alias_user"`
 	Key    string `gorm:"not null; index:idx_alias_type_key"`

models/heartbeat.go

@@ -7,7 +7,7 @@ import (
 
 type Heartbeat struct {
 	ID              uint       `gorm:"primary_key"`
-	User            *User      `json:"-" gorm:"not null"`
+	User            *User      `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
 	UserID          string     `json:"-" gorm:"not null; index:idx_time_user"`
 	Entity          string     `json:"entity" gorm:"not null; index:idx_entity"`
 	Type            string     `json:"type"`

models/language_mapping.go

@@ -2,7 +2,7 @@ package models
 
 type LanguageMapping struct {
 	ID        uint   `json:"id" gorm:"primary_key"`
-	User      *User  `json:"-" gorm:"not null"`
+	User      *User  `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
 	UserID    string `json:"-" gorm:"not null; index:idx_language_mapping_user; uniqueIndex:idx_language_mapping_composite"`
 	Extension string `json:"extension" gorm:"uniqueIndex:idx_language_mapping_composite; type:varchar(16)"`
 	Language  string `json:"language" gorm:"type:varchar(64)"`

models/summary.go

@@ -35,20 +35,20 @@ const UnknownSummaryKey = "unknown"
 
 type Summary struct {
 	ID               uint           `json:"-" gorm:"primary_key"`
-	User             *User          `json:"-" gorm:"not null"`
+	User             *User          `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
 	UserID           string         `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
 	FromTime         CustomTime     `json:"from" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
 	ToTime           CustomTime     `json:"to" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
-	Projects         []*SummaryItem `json:"projects"`
+	Projects         []*SummaryItem `json:"projects" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
-	Languages        []*SummaryItem `json:"languages"`
+	Languages        []*SummaryItem `json:"languages" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
-	Editors          []*SummaryItem `json:"editors"`
+	Editors          []*SummaryItem `json:"editors" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
-	OperatingSystems []*SummaryItem `json:"operating_systems"`
+	OperatingSystems []*SummaryItem `json:"operating_systems" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
-	Machines         []*SummaryItem `json:"machines"`
+	Machines         []*SummaryItem `json:"machines" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
 }
 
 type SummaryItem struct {
 	ID        uint          `json:"-" gorm:"primary_key"`
-	Summary   *Summary      `json:"-" gorm:"not null"`
+	Summary   *Summary      `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
 	SummaryID uint          `json:"-"`
 	Type      uint8         `json:"-"`
 	Key       string        `json:"key"`

models/user.go

@@ -1,12 +1,12 @@
 package models
 
 type User struct {
-	ID             string     `json:"id" gorm:"primary_key"`
+	ID             string       `json:"id" gorm:"primary_key"`
-	ApiKey         string     `json:"api_key" gorm:"unique"`
+	ApiKey         string       `json:"api_key" gorm:"unique"`
-	Password       string     `json:"-"`
+	Password       string       `json:"-"`
-	CreatedAt      CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
+	CreatedAt      CustomTime   `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
-	LastLoggedInAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
+	LastLoggedInAt CustomTime   `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
-	BadgesEnabled  bool       `json:"-" gorm:"default:false; type:bool"`
+	BadgesEnabled  bool         `json:"-" gorm:"default:false; type:bool"`
 }
 
 type Login struct {

repositories/summary.go

@@ -51,3 +51,12 @@ func (r *SummaryRepository) GetLatestByUser() ([]*models.Summary, error) {
 	}
 	return summaries, nil
 }
+
+func (r *SummaryRepository) DeleteByUser(userId string) error {
+	if err := r.db.
+		Where("user_id = ?", userId).
+		Delete(models.Summary{}).Error; err != nil {
+		return err
+	}
+	return nil
+}

routes/settings.go

@@ -7,6 +7,7 @@ import (
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/services"
 	"github.com/muety/wakapi/utils"
+	"log"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -15,14 +16,18 @@ import (
 type SettingsHandler struct {
 	config              *conf.Config
 	userSrvc            *services.UserService
+	summarySrvc         *services.SummaryService
+	aggregationSrvc     *services.AggregationService
 	languageMappingSrvc *services.LanguageMappingService
 }
 
 var credentialsDecoder = schema.NewDecoder()
 
-func NewSettingsHandler(userService *services.UserService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
+func NewSettingsHandler(userService *services.UserService, summaryService *services.SummaryService, aggregationService *services.AggregationService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
 	return &SettingsHandler{
 		config:              conf.Get(),
+		summarySrvc:         summaryService,
+		aggregationSrvc:     aggregationService,
 		languageMappingSrvc: languageMappingService,
 		userSrvc:            userService,
 	}
@@ -133,7 +138,7 @@ func (h *SettingsHandler) DeleteLanguageMapping(w http.ResponseWriter, r *http.R
 	http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("mapping deleted successfully")), http.StatusFound)
 }
 
-func (h *SettingsHandler) PostCreateLanguageMapping(w http.ResponseWriter, r *http.Request) {
+func (h *SettingsHandler) PostLanguageMapping(w http.ResponseWriter, r *http.Request) {
 	if h.config.IsDev() {
 		loadTemplates()
 	}
@@ -180,7 +185,6 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
 	}
 
 	user := r.Context().Value(models.UserKey).(*models.User)
-
 	if _, err := h.userSrvc.ToggleBadges(user); err != nil {
 		http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("internal server error")), http.StatusFound)
 		return
@@ -188,3 +192,26 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
 
 	http.Redirect(w, r, fmt.Sprintf("%s/settings", h.config.Server.BasePath), http.StatusFound)
 }
+
+func (h *SettingsHandler) PostRegenerateSummaries(w http.ResponseWriter, r *http.Request) {
+	if h.config.IsDev() {
+		loadTemplates()
+	}
+
+	user := r.Context().Value(models.UserKey).(*models.User)
+
+	log.Printf("clearing summaries for user '%s'\n", user.ID)
+	if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+		log.Printf("failed to clear summaries: %v\n", err)
+		http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to delete old summaries")), http.StatusFound)
+		return
+	}
+
+	if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
+		log.Printf("failed to regenerate summaries: %v\n", err)
+		http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to generate aggregations")), http.StatusFound)
+		return
+	}
+
+	http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("summaries are being regenerated – this may take a few second")), http.StatusFound)
+}

services/aggregation.go

@@ -38,10 +38,18 @@ type AggregationJob struct {
 
 // Schedule a job to (re-)generate summaries every day shortly after midnight
 func (srv *AggregationService) Schedule() {
+	// Run once initially
+	if err := srv.Run(nil); err != nil {
+		log.Fatalf("failed to run aggregation jobs: %v\n", err)
+	}
+
+	gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.Run, nil)
+	<-gocron.Start()
+}
+
+func (srv *AggregationService) Run(userIds map[string]bool) error {
 	jobs := make(chan *AggregationJob)
 	summaries := make(chan *models.Summary)
-	defer close(jobs)
-	defer close(summaries)
 
 	for i := 0; i < runtime.NumCPU(); i++ {
 		go srv.summaryWorker(jobs, summaries)
@@ -51,11 +59,7 @@ func (srv *AggregationService) Schedule() {
 		go srv.persistWorker(summaries)
 	}
 
-	// Run once initially
+	return srv.trigger(jobs, userIds)
-	srv.trigger(jobs)
-
-	gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.trigger, jobs)
-	<-gocron.Start()
 }
 
 func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) {
@@ -77,13 +81,22 @@ func (srv *AggregationService) persistWorker(summaries <-chan *models.Summary) {
 	}
 }
 
-func (srv *AggregationService) trigger(jobs chan<- *AggregationJob) error {
+func (srv *AggregationService) trigger(jobs chan<- *AggregationJob, userIds map[string]bool) error {
 	log.Println("Generating summaries.")
 
-	users, err := srv.userService.GetAll()
+	var users []*models.User
-	if err != nil {
+	if allUsers, err := srv.userService.GetAll(); err != nil {
 		log.Println(err)
 		return err
+	} else if userIds != nil && len(userIds) > 0 {
+		users = make([]*models.User, len(userIds))
+		for i, u := range allUsers {
+			if yes, ok := userIds[u.ID]; yes && ok {
+				users[i] = u
+			}
+		}
+	} else {
+		users = allUsers
 	}
 
 	latestSummaries, err := srv.summaryService.GetLatestByUser()

services/summary.go

@@ -225,6 +225,10 @@ func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) {
 	return srv.repository.GetLatestByUser()
 }
 
+func (srv *SummaryService) DeleteByUser(userId string) error {
+	return srv.repository.DeleteByUser(userId)
+}
+
 func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, user *models.User, c chan models.SummaryItemContainer) {
 	durations := make(map[string]time.Duration)
 

utils/common.go

@@ -25,4 +25,4 @@ func ParseUserAgent(ua string) (string, string, error) {
 		return "", "", errors.New("failed to parse user agent string")
 	}
 	return groups[0][1], groups[0][2], nil
-}
+}

version.txt

@@ -1 +1 @@
-1.14.1
+1.15.0

views/settings.tpl.html

@@ -48,7 +48,7 @@
             </form>
         </div>
 
-        <div class="w-full mt-4 mb-8 pb-8">
+        <div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
             <div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
                 Reset API Key
             </div>
@@ -66,7 +66,7 @@
             </form>
         </div>
 
-        <div class="w-full mt-4 mb-8 pb-8">
+        <div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
             <div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
                 Language Mappings
             </div>
@@ -119,7 +119,7 @@
 
         </div>
 
-        <div class="w-full mt-4 mb-8 pb-8">
+        <div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
             <div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
                 Badges
             </div>
@@ -169,6 +169,34 @@
                 </div>
             </form>
         </div>
+
+        <div class="w-full mt-4 mb-8 pb-8">
+            <div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
+                ⚠️ Danger Zone
+            </div>
+            <div class="mt-10 text-gray-300 text-sm">
+                <h3 class="font-semibold text-md mb-4 border-b border-green-700 inline-block">
+                    Regenerate summaries
+                </h3>
+                <p>
+                    Wakapi improves its efficiency and speed by automatically aggregating individual heartbeats to summaries on a per-day basis.
+                    That is, historic summaries, i.e. such from past days, are generated once and only fetched from the database in a static fashion afterwards, unless you pass <span class="font-mono font-normal bg-gray-900 p-1 rounded whitespace-no-wrap">&recompute=true</span> with your request.
+                </p>
+                <p class="mt-2">
+                    If, for some reason, these aggregated summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively), you may want to re-generate them from raw heartbeats.
+                </p>
+                <p class="mt-2">
+                    <strong>Note:</strong> Only run this action if you know what you are doing. Data might be lost is case heartbeats were deleted after the respective summaries had been generated.
+                </p>
+            </div>
+            <div class="mt-10 flex justify-center">
+                <form action="settings/regenerate" method="post" id="form-regenerate-summaries">
+                    <button type="button" class="py-1 px-3 rounded bg-red-500 hover:bg-red-600 text-white text-sm" id="btn-regenerate-summaries">
+                        Clear & Regenerate
+                    </button>
+                </form>
+            </div>
+        </div>
     </div>
 </main>
 
@@ -182,6 +210,14 @@
         e.innerHTML = e.innerHTML.replace('%s', baseUrl)
         e.classList.remove('hidden')
     })
+
+    const btnRegenerate = document.querySelector("#btn-regenerate-summaries")
+    const formRegenerate = document.querySelector('#form-regenerate-summaries')
+    btnRegenerate.addEventListener('click', () => {
+        if (confirm('Are you sure?')) {
+            formRegenerate.submit()
+        }
+    })
 </script>
 
 {{ template "footer.tpl.html" . }}