diff --git a/main.go b/main.go index 0538b49..631b20d 100644 --- a/main.go +++ b/main.go @@ -63,6 +63,9 @@ func main() { db.Raw("PRAGMA foreign_keys = ON;") } + if config.IsDev() { + db = db.Debug() + } sqlDb, _ := db.DB() sqlDb.SetMaxIdleConns(int(config.Db.MaxConn)) sqlDb.SetMaxOpenConns(int(config.Db.MaxConn)) @@ -103,7 +106,7 @@ func main() { summaryHandler := routes.NewSummaryHandler(summaryService) healthHandler := routes.NewHealthHandler(db) heartbeatHandler := routes.NewHeartbeatHandler(heartbeatService, languageMappingService) - settingsHandler := routes.NewSettingsHandler(userService, languageMappingService) + settingsHandler := routes.NewSettingsHandler(userService, summaryService, aggregationService, languageMappingService) publicHandler := routes.NewIndexHandler(userService, keyValueService) wakatimeV1AllHandler := wtV1Routes.NewAllTimeHandler(summaryService) wakatimeV1SummariesHandler := wtV1Routes.NewSummariesHandler(summaryService) @@ -148,10 +151,11 @@ func main() { // Settings Routes settingsRouter.Methods(http.MethodGet).HandlerFunc(settingsHandler.GetIndex) settingsRouter.Path("/credentials").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCredentials) - settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCreateLanguageMapping) + settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostLanguageMapping) settingsRouter.Path("/language_mappings/delete").Methods(http.MethodPost).HandlerFunc(settingsHandler.DeleteLanguageMapping) settingsRouter.Path("/reset").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostResetApiKey) settingsRouter.Path("/badges").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostToggleBadges) + settingsRouter.Path("/regenerate").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostRegenerateSummaries) // API Routes apiRouter.Path("/heartbeat").Methods(http.MethodPost).HandlerFunc(heartbeatHandler.ApiPost) diff --git a/migrations/common/custom_pre.go b/migrations/common/custom_pre.go index 4e53aca..f076945 100644 --- a/migrations/common/custom_pre.go +++ b/migrations/common/custom_pre.go @@ -30,6 +30,71 @@ func init() { }, name: "rename language mappings table", }, + { + f: func(db *gorm.DB, cfg *config.Config) error { + // drop all already existing foreign key constraints + // afterwards let them be re-created by auto migrate with the newly introduced cascade settings, + + migrator := db.Migrator() + const lookupKey = "20201106-migration_cascade_constraints" + + if cfg.Db.Dialect == "sqlite3" { + // https://stackoverflow.com/a/1884893/3112139 + // unfortunately, we can't migrate existing sqlite databases to the newly introduced cascade settings + // things like deleting all summaries won't work in those cases unless an entirely new db is created + log.Println("not attempting to drop and regenerate constraints on sqlite") + return nil + } + + if !migrator.HasTable(&models.KeyStringValue{}) { + log.Println("key-value table not yet existing") + return nil + } + + condition := "key = ?" + if cfg.Db.Dialect == "mysql" { + condition = "`key` = ?" + } + lookupResult := db.Where(condition, lookupKey).First(&models.KeyStringValue{}) + if lookupResult.Error == nil && lookupResult.RowsAffected > 0 { + log.Println("no need to migrate cascade constraints") + return nil + } + + // SELECT * FROM INFORMATION_SCHEMA.table_constraints; + constraints := map[string]interface{}{ + "fk_summaries_editors": &models.SummaryItem{}, + "fk_summaries_languages": &models.SummaryItem{}, + "fk_summaries_machines": &models.SummaryItem{}, + "fk_summaries_operating_systems": &models.SummaryItem{}, + "fk_summaries_projects": &models.SummaryItem{}, + "fk_summary_items_summary": &models.SummaryItem{}, + "fk_summaries_user": &models.Summary{}, + "fk_language_mappings_user": &models.LanguageMapping{}, + "fk_heartbeats_user": &models.Heartbeat{}, + "fk_aliases_user": &models.Alias{}, + } + + for name, table := range constraints { + if migrator.HasConstraint(table, name) { + log.Printf("dropping constraint '%s'", name) + if err := migrator.DropConstraint(table, name); err != nil { + return err + } + } + } + + if err := db.Create(&models.KeyStringValue{ + Key: lookupKey, + Value: "done", + }).Error; err != nil { + return err + } + + return nil + }, + name: "add cascade constraints", + }, } } diff --git a/models/alias.go b/models/alias.go index 4f121b2..a6136ba 100644 --- a/models/alias.go +++ b/models/alias.go @@ -2,7 +2,7 @@ package models type Alias struct { ID uint `gorm:"primary_key"` - Type uint8 `gorm:"not null; index:idx_alias_type_key"` + Type uint8 `gorm:"not null; index:idx_alias_type_key; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` User *User `json:"-" gorm:"not null"` UserID string `gorm:"not null; index:idx_alias_user"` Key string `gorm:"not null; index:idx_alias_type_key"` diff --git a/models/heartbeat.go b/models/heartbeat.go index f9e434b..e3be990 100644 --- a/models/heartbeat.go +++ b/models/heartbeat.go @@ -7,7 +7,7 @@ import ( type Heartbeat struct { ID uint `gorm:"primary_key"` - User *User `json:"-" gorm:"not null"` + User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"` UserID string `json:"-" gorm:"not null; index:idx_time_user"` Entity string `json:"entity" gorm:"not null; index:idx_entity"` Type string `json:"type"` diff --git a/models/language_mapping.go b/models/language_mapping.go index e0aee9d..371cf9a 100644 --- a/models/language_mapping.go +++ b/models/language_mapping.go @@ -2,7 +2,7 @@ package models type LanguageMapping struct { ID uint `json:"id" gorm:"primary_key"` - User *User `json:"-" gorm:"not null"` + User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` UserID string `json:"-" gorm:"not null; index:idx_language_mapping_user; uniqueIndex:idx_language_mapping_composite"` Extension string `json:"extension" gorm:"uniqueIndex:idx_language_mapping_composite; type:varchar(16)"` Language string `json:"language" gorm:"type:varchar(64)"` diff --git a/models/summary.go b/models/summary.go index b8ba8d5..fc44193 100644 --- a/models/summary.go +++ b/models/summary.go @@ -35,20 +35,20 @@ const UnknownSummaryKey = "unknown" type Summary struct { ID uint `json:"-" gorm:"primary_key"` - User *User `json:"-" gorm:"not null"` + User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` UserID string `json:"user_id" gorm:"not null; index:idx_time_summary_user"` FromTime CustomTime `json:"from" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"` ToTime CustomTime `json:"to" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"` - Projects []*SummaryItem `json:"projects"` - Languages []*SummaryItem `json:"languages"` - Editors []*SummaryItem `json:"editors"` - OperatingSystems []*SummaryItem `json:"operating_systems"` - Machines []*SummaryItem `json:"machines"` + Projects []*SummaryItem `json:"projects" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` + Languages []*SummaryItem `json:"languages" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` + Editors []*SummaryItem `json:"editors" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` + OperatingSystems []*SummaryItem `json:"operating_systems" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` + Machines []*SummaryItem `json:"machines" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` } type SummaryItem struct { ID uint `json:"-" gorm:"primary_key"` - Summary *Summary `json:"-" gorm:"not null"` + Summary *Summary `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` SummaryID uint `json:"-"` Type uint8 `json:"-"` Key string `json:"key"` diff --git a/models/user.go b/models/user.go index e99182a..1c4d500 100644 --- a/models/user.go +++ b/models/user.go @@ -1,12 +1,12 @@ package models type User struct { - ID string `json:"id" gorm:"primary_key"` - ApiKey string `json:"api_key" gorm:"unique"` - Password string `json:"-"` - CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"` - LastLoggedInAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"` - BadgesEnabled bool `json:"-" gorm:"default:false; type:bool"` + ID string `json:"id" gorm:"primary_key"` + ApiKey string `json:"api_key" gorm:"unique"` + Password string `json:"-"` + CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"` + LastLoggedInAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"` + BadgesEnabled bool `json:"-" gorm:"default:false; type:bool"` } type Login struct { diff --git a/repositories/summary.go b/repositories/summary.go index 39aaa9e..d7c4dca 100644 --- a/repositories/summary.go +++ b/repositories/summary.go @@ -51,3 +51,12 @@ func (r *SummaryRepository) GetLatestByUser() ([]*models.Summary, error) { } return summaries, nil } + +func (r *SummaryRepository) DeleteByUser(userId string) error { + if err := r.db. + Where("user_id = ?", userId). + Delete(models.Summary{}).Error; err != nil { + return err + } + return nil +} \ No newline at end of file diff --git a/routes/settings.go b/routes/settings.go index c9e9a33..07f9c35 100644 --- a/routes/settings.go +++ b/routes/settings.go @@ -7,6 +7,7 @@ import ( "github.com/muety/wakapi/models" "github.com/muety/wakapi/services" "github.com/muety/wakapi/utils" + "log" "net/http" "net/url" "strconv" @@ -15,14 +16,18 @@ import ( type SettingsHandler struct { config *conf.Config userSrvc *services.UserService + summarySrvc *services.SummaryService + aggregationSrvc *services.AggregationService languageMappingSrvc *services.LanguageMappingService } var credentialsDecoder = schema.NewDecoder() -func NewSettingsHandler(userService *services.UserService, languageMappingService *services.LanguageMappingService) *SettingsHandler { +func NewSettingsHandler(userService *services.UserService, summaryService *services.SummaryService, aggregationService *services.AggregationService, languageMappingService *services.LanguageMappingService) *SettingsHandler { return &SettingsHandler{ config: conf.Get(), + summarySrvc: summaryService, + aggregationSrvc: aggregationService, languageMappingSrvc: languageMappingService, userSrvc: userService, } @@ -133,7 +138,7 @@ func (h *SettingsHandler) DeleteLanguageMapping(w http.ResponseWriter, r *http.R http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("mapping deleted successfully")), http.StatusFound) } -func (h *SettingsHandler) PostCreateLanguageMapping(w http.ResponseWriter, r *http.Request) { +func (h *SettingsHandler) PostLanguageMapping(w http.ResponseWriter, r *http.Request) { if h.config.IsDev() { loadTemplates() } @@ -180,7 +185,6 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques } user := r.Context().Value(models.UserKey).(*models.User) - if _, err := h.userSrvc.ToggleBadges(user); err != nil { http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("internal server error")), http.StatusFound) return @@ -188,3 +192,26 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques http.Redirect(w, r, fmt.Sprintf("%s/settings", h.config.Server.BasePath), http.StatusFound) } + +func (h *SettingsHandler) PostRegenerateSummaries(w http.ResponseWriter, r *http.Request) { + if h.config.IsDev() { + loadTemplates() + } + + user := r.Context().Value(models.UserKey).(*models.User) + + log.Printf("clearing summaries for user '%s'\n", user.ID) + if err := h.summarySrvc.DeleteByUser(user.ID); err != nil { + log.Printf("failed to clear summaries: %v\n", err) + http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to delete old summaries")), http.StatusFound) + return + } + + if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil { + log.Printf("failed to regenerate summaries: %v\n", err) + http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to generate aggregations")), http.StatusFound) + return + } + + http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("summaries are being regenerated – this may take a few second")), http.StatusFound) +} diff --git a/services/aggregation.go b/services/aggregation.go index 0f14b6f..596bb3d 100644 --- a/services/aggregation.go +++ b/services/aggregation.go @@ -38,10 +38,18 @@ type AggregationJob struct { // Schedule a job to (re-)generate summaries every day shortly after midnight func (srv *AggregationService) Schedule() { + // Run once initially + if err := srv.Run(nil); err != nil { + log.Fatalf("failed to run aggregation jobs: %v\n", err) + } + + gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.Run, nil) + <-gocron.Start() +} + +func (srv *AggregationService) Run(userIds map[string]bool) error { jobs := make(chan *AggregationJob) summaries := make(chan *models.Summary) - defer close(jobs) - defer close(summaries) for i := 0; i < runtime.NumCPU(); i++ { go srv.summaryWorker(jobs, summaries) @@ -51,11 +59,7 @@ func (srv *AggregationService) Schedule() { go srv.persistWorker(summaries) } - // Run once initially - srv.trigger(jobs) - - gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.trigger, jobs) - <-gocron.Start() + return srv.trigger(jobs, userIds) } func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) { @@ -77,13 +81,22 @@ func (srv *AggregationService) persistWorker(summaries <-chan *models.Summary) { } } -func (srv *AggregationService) trigger(jobs chan<- *AggregationJob) error { +func (srv *AggregationService) trigger(jobs chan<- *AggregationJob, userIds map[string]bool) error { log.Println("Generating summaries.") - users, err := srv.userService.GetAll() - if err != nil { + var users []*models.User + if allUsers, err := srv.userService.GetAll(); err != nil { log.Println(err) return err + } else if userIds != nil && len(userIds) > 0 { + users = make([]*models.User, len(userIds)) + for i, u := range allUsers { + if yes, ok := userIds[u.ID]; yes && ok { + users[i] = u + } + } + } else { + users = allUsers } latestSummaries, err := srv.summaryService.GetLatestByUser() diff --git a/services/summary.go b/services/summary.go index d6248c4..dd27ed0 100644 --- a/services/summary.go +++ b/services/summary.go @@ -225,6 +225,10 @@ func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) { return srv.repository.GetLatestByUser() } +func (srv *SummaryService) DeleteByUser(userId string) error { + return srv.repository.DeleteByUser(userId) +} + func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, user *models.User, c chan models.SummaryItemContainer) { durations := make(map[string]time.Duration) diff --git a/utils/common.go b/utils/common.go index f46d92d..05347df 100644 --- a/utils/common.go +++ b/utils/common.go @@ -25,4 +25,4 @@ func ParseUserAgent(ua string) (string, string, error) { return "", "", errors.New("failed to parse user agent string") } return groups[0][1], groups[0][2], nil -} +} \ No newline at end of file diff --git a/version.txt b/version.txt index 63e799c..141f2e8 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.14.1 +1.15.0 diff --git a/views/settings.tpl.html b/views/settings.tpl.html index 2b8ed68..72217ea 100644 --- a/views/settings.tpl.html +++ b/views/settings.tpl.html @@ -48,7 +48,7 @@ -
+
Reset API Key
@@ -66,7 +66,7 @@
-
+
Language Mappings
@@ -119,7 +119,7 @@
-
+
Badges
@@ -169,6 +169,34 @@
+ +
+
+ ⚠️ Danger Zone +
+
+

+ Regenerate summaries +

+

+ Wakapi improves its efficiency and speed by automatically aggregating individual heartbeats to summaries on a per-day basis. + That is, historic summaries, i.e. such from past days, are generated once and only fetched from the database in a static fashion afterwards, unless you pass &recompute=true with your request. +

+

+ If, for some reason, these aggregated summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively), you may want to re-generate them from raw heartbeats. +

+

+ Note: Only run this action if you know what you are doing. Data might be lost is case heartbeats were deleted after the respective summaries had been generated. +

+
+
+
+ +
+
+
@@ -182,6 +210,14 @@ e.innerHTML = e.innerHTML.replace('%s', baseUrl) e.classList.remove('hidden') }) + + const btnRegenerate = document.querySelector("#btn-regenerate-summaries") + const formRegenerate = document.querySelector('#form-regenerate-summaries') + btnRegenerate.addEventListener('click', () => { + if (confirm('Are you sure?')) { + formRegenerate.submit() + } + }) {{ template "footer.tpl.html" . }}