mirror of https://github.com/muety/wakapi.git
feat: add ability to regenerate summaries
fix: database cascade settings chore: debug log mode for gorm queries is back
This commit is contained in:
Ferdinand Mütsch
parent
e6a04cc76d
commit
e269b37b0e
8
main.go
8
main.go
|
|
@ -63,6 +63,9 @@ func main() {
|
|||
db.Raw("PRAGMA foreign_keys = ON;")
|
||||
}
|
||||
|
||||
if config.IsDev() {
|
||||
db = db.Debug()
|
||||
}
|
||||
sqlDb, _ := db.DB()
|
||||
sqlDb.SetMaxIdleConns(int(config.Db.MaxConn))
|
||||
sqlDb.SetMaxOpenConns(int(config.Db.MaxConn))
|
||||
|
|
@ -103,7 +106,7 @@ func main() {
|
|||
summaryHandler := routes.NewSummaryHandler(summaryService)
|
||||
healthHandler := routes.NewHealthHandler(db)
|
||||
heartbeatHandler := routes.NewHeartbeatHandler(heartbeatService, languageMappingService)
|
||||
settingsHandler := routes.NewSettingsHandler(userService, languageMappingService)
|
||||
settingsHandler := routes.NewSettingsHandler(userService, summaryService, aggregationService, languageMappingService)
|
||||
publicHandler := routes.NewIndexHandler(userService, keyValueService)
|
||||
wakatimeV1AllHandler := wtV1Routes.NewAllTimeHandler(summaryService)
|
||||
wakatimeV1SummariesHandler := wtV1Routes.NewSummariesHandler(summaryService)
|
||||
|
|
@ -148,10 +151,11 @@ func main() {
|
|||
// Settings Routes
|
||||
settingsRouter.Methods(http.MethodGet).HandlerFunc(settingsHandler.GetIndex)
|
||||
settingsRouter.Path("/credentials").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCredentials)
|
||||
settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCreateLanguageMapping)
|
||||
settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostLanguageMapping)
|
||||
settingsRouter.Path("/language_mappings/delete").Methods(http.MethodPost).HandlerFunc(settingsHandler.DeleteLanguageMapping)
|
||||
settingsRouter.Path("/reset").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostResetApiKey)
|
||||
settingsRouter.Path("/badges").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostToggleBadges)
|
||||
settingsRouter.Path("/regenerate").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostRegenerateSummaries)
|
||||
|
||||
// API Routes
|
||||
apiRouter.Path("/heartbeat").Methods(http.MethodPost).HandlerFunc(heartbeatHandler.ApiPost)
|
||||
|
|
|
|||
|
|
@ -30,6 +30,71 @@ func init() {
|
|||
},
|
||||
name: "rename language mappings table",
|
||||
},
|
||||
{
|
||||
f: func(db *gorm.DB, cfg *config.Config) error {
|
||||
// drop all already existing foreign key constraints
|
||||
// afterwards let them be re-created by auto migrate with the newly introduced cascade settings,
|
||||
|
||||
migrator := db.Migrator()
|
||||
const lookupKey = "20201106-migration_cascade_constraints"
|
||||
|
||||
if cfg.Db.Dialect == "sqlite3" {
|
||||
// https://stackoverflow.com/a/1884893/3112139
|
||||
// unfortunately, we can't migrate existing sqlite databases to the newly introduced cascade settings
|
||||
// things like deleting all summaries won't work in those cases unless an entirely new db is created
|
||||
log.Println("not attempting to drop and regenerate constraints on sqlite")
|
||||
return nil
|
||||
}
|
||||
|
||||
if !migrator.HasTable(&models.KeyStringValue{}) {
|
||||
log.Println("key-value table not yet existing")
|
||||
return nil
|
||||
}
|
||||
|
||||
condition := "key = ?"
|
||||
if cfg.Db.Dialect == "mysql" {
|
||||
condition = "`key` = ?"
|
||||
}
|
||||
lookupResult := db.Where(condition, lookupKey).First(&models.KeyStringValue{})
|
||||
if lookupResult.Error == nil && lookupResult.RowsAffected > 0 {
|
||||
log.Println("no need to migrate cascade constraints")
|
||||
return nil
|
||||
}
|
||||
|
||||
// SELECT * FROM INFORMATION_SCHEMA.table_constraints;
|
||||
constraints := map[string]interface{}{
|
||||
"fk_summaries_editors": &models.SummaryItem{},
|
||||
"fk_summaries_languages": &models.SummaryItem{},
|
||||
"fk_summaries_machines": &models.SummaryItem{},
|
||||
"fk_summaries_operating_systems": &models.SummaryItem{},
|
||||
"fk_summaries_projects": &models.SummaryItem{},
|
||||
"fk_summary_items_summary": &models.SummaryItem{},
|
||||
"fk_summaries_user": &models.Summary{},
|
||||
"fk_language_mappings_user": &models.LanguageMapping{},
|
||||
"fk_heartbeats_user": &models.Heartbeat{},
|
||||
"fk_aliases_user": &models.Alias{},
|
||||
}
|
||||
|
||||
for name, table := range constraints {
|
||||
if migrator.HasConstraint(table, name) {
|
||||
log.Printf("dropping constraint '%s'", name)
|
||||
if err := migrator.DropConstraint(table, name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := db.Create(&models.KeyStringValue{
|
||||
Key: lookupKey,
|
||||
Value: "done",
|
||||
}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
name: "add cascade constraints",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package models
|
|||
|
||||
type Alias struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
Type uint8 `gorm:"not null; index:idx_alias_type_key"`
|
||||
Type uint8 `gorm:"not null; index:idx_alias_type_key; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
User *User `json:"-" gorm:"not null"`
|
||||
UserID string `gorm:"not null; index:idx_alias_user"`
|
||||
Key string `gorm:"not null; index:idx_alias_type_key"`
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import (
|
|||
|
||||
type Heartbeat struct {
|
||||
ID uint `gorm:"primary_key"`
|
||||
User *User `json:"-" gorm:"not null"`
|
||||
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
|
||||
UserID string `json:"-" gorm:"not null; index:idx_time_user"`
|
||||
Entity string `json:"entity" gorm:"not null; index:idx_entity"`
|
||||
Type string `json:"type"`
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ package models
|
|||
|
||||
type LanguageMapping struct {
|
||||
ID uint `json:"id" gorm:"primary_key"`
|
||||
User *User `json:"-" gorm:"not null"`
|
||||
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
UserID string `json:"-" gorm:"not null; index:idx_language_mapping_user; uniqueIndex:idx_language_mapping_composite"`
|
||||
Extension string `json:"extension" gorm:"uniqueIndex:idx_language_mapping_composite; type:varchar(16)"`
|
||||
Language string `json:"language" gorm:"type:varchar(64)"`
|
||||
|
|
|
|||
|
|
@ -35,20 +35,20 @@ const UnknownSummaryKey = "unknown"
|
|||
|
||||
type Summary struct {
|
||||
ID uint `json:"-" gorm:"primary_key"`
|
||||
User *User `json:"-" gorm:"not null"`
|
||||
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
UserID string `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
|
||||
FromTime CustomTime `json:"from" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
|
||||
ToTime CustomTime `json:"to" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
|
||||
Projects []*SummaryItem `json:"projects"`
|
||||
Languages []*SummaryItem `json:"languages"`
|
||||
Editors []*SummaryItem `json:"editors"`
|
||||
OperatingSystems []*SummaryItem `json:"operating_systems"`
|
||||
Machines []*SummaryItem `json:"machines"`
|
||||
Projects []*SummaryItem `json:"projects" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Languages []*SummaryItem `json:"languages" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Editors []*SummaryItem `json:"editors" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
OperatingSystems []*SummaryItem `json:"operating_systems" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
Machines []*SummaryItem `json:"machines" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
}
|
||||
|
||||
type SummaryItem struct {
|
||||
ID uint `json:"-" gorm:"primary_key"`
|
||||
Summary *Summary `json:"-" gorm:"not null"`
|
||||
Summary *Summary `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
|
||||
SummaryID uint `json:"-"`
|
||||
Type uint8 `json:"-"`
|
||||
Key string `json:"key"`
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
package models
|
||||
|
||||
type User struct {
|
||||
ID string `json:"id" gorm:"primary_key"`
|
||||
ApiKey string `json:"api_key" gorm:"unique"`
|
||||
Password string `json:"-"`
|
||||
CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
|
||||
LastLoggedInAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
|
||||
BadgesEnabled bool `json:"-" gorm:"default:false; type:bool"`
|
||||
ID string `json:"id" gorm:"primary_key"`
|
||||
ApiKey string `json:"api_key" gorm:"unique"`
|
||||
Password string `json:"-"`
|
||||
CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
|
||||
LastLoggedInAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP"`
|
||||
BadgesEnabled bool `json:"-" gorm:"default:false; type:bool"`
|
||||
}
|
||||
|
||||
type Login struct {
|
||||
|
|
|
|||
|
|
@ -51,3 +51,12 @@ func (r *SummaryRepository) GetLatestByUser() ([]*models.Summary, error) {
|
|||
}
|
||||
return summaries, nil
|
||||
}
|
||||
|
||||
func (r *SummaryRepository) DeleteByUser(userId string) error {
|
||||
if err := r.db.
|
||||
Where("user_id = ?", userId).
|
||||
Delete(models.Summary{}).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -7,6 +7,7 @@ import (
|
|||
"github.com/muety/wakapi/models"
|
||||
"github.com/muety/wakapi/services"
|
||||
"github.com/muety/wakapi/utils"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
|
|
@ -15,14 +16,18 @@ import (
|
|||
type SettingsHandler struct {
|
||||
config *conf.Config
|
||||
userSrvc *services.UserService
|
||||
summarySrvc *services.SummaryService
|
||||
aggregationSrvc *services.AggregationService
|
||||
languageMappingSrvc *services.LanguageMappingService
|
||||
}
|
||||
|
||||
var credentialsDecoder = schema.NewDecoder()
|
||||
|
||||
func NewSettingsHandler(userService *services.UserService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
|
||||
func NewSettingsHandler(userService *services.UserService, summaryService *services.SummaryService, aggregationService *services.AggregationService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
|
||||
return &SettingsHandler{
|
||||
config: conf.Get(),
|
||||
summarySrvc: summaryService,
|
||||
aggregationSrvc: aggregationService,
|
||||
languageMappingSrvc: languageMappingService,
|
||||
userSrvc: userService,
|
||||
}
|
||||
|
|
@ -133,7 +138,7 @@ func (h *SettingsHandler) DeleteLanguageMapping(w http.ResponseWriter, r *http.R
|
|||
http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("mapping deleted successfully")), http.StatusFound)
|
||||
}
|
||||
|
||||
func (h *SettingsHandler) PostCreateLanguageMapping(w http.ResponseWriter, r *http.Request) {
|
||||
func (h *SettingsHandler) PostLanguageMapping(w http.ResponseWriter, r *http.Request) {
|
||||
if h.config.IsDev() {
|
||||
loadTemplates()
|
||||
}
|
||||
|
|
@ -180,7 +185,6 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
|
|||
}
|
||||
|
||||
user := r.Context().Value(models.UserKey).(*models.User)
|
||||
|
||||
if _, err := h.userSrvc.ToggleBadges(user); err != nil {
|
||||
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("internal server error")), http.StatusFound)
|
||||
return
|
||||
|
|
@ -188,3 +192,26 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
|
|||
|
||||
http.Redirect(w, r, fmt.Sprintf("%s/settings", h.config.Server.BasePath), http.StatusFound)
|
||||
}
|
||||
|
||||
func (h *SettingsHandler) PostRegenerateSummaries(w http.ResponseWriter, r *http.Request) {
|
||||
if h.config.IsDev() {
|
||||
loadTemplates()
|
||||
}
|
||||
|
||||
user := r.Context().Value(models.UserKey).(*models.User)
|
||||
|
||||
log.Printf("clearing summaries for user '%s'\n", user.ID)
|
||||
if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
|
||||
log.Printf("failed to clear summaries: %v\n", err)
|
||||
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to delete old summaries")), http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
|
||||
log.Printf("failed to regenerate summaries: %v\n", err)
|
||||
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to generate aggregations")), http.StatusFound)
|
||||
return
|
||||
}
|
||||
|
||||
http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("summaries are being regenerated – this may take a few second")), http.StatusFound)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,10 +38,18 @@ type AggregationJob struct {
|
|||
|
||||
// Schedule a job to (re-)generate summaries every day shortly after midnight
|
||||
func (srv *AggregationService) Schedule() {
|
||||
// Run once initially
|
||||
if err := srv.Run(nil); err != nil {
|
||||
log.Fatalf("failed to run aggregation jobs: %v\n", err)
|
||||
}
|
||||
|
||||
gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.Run, nil)
|
||||
<-gocron.Start()
|
||||
}
|
||||
|
||||
func (srv *AggregationService) Run(userIds map[string]bool) error {
|
||||
jobs := make(chan *AggregationJob)
|
||||
summaries := make(chan *models.Summary)
|
||||
defer close(jobs)
|
||||
defer close(summaries)
|
||||
|
||||
for i := 0; i < runtime.NumCPU(); i++ {
|
||||
go srv.summaryWorker(jobs, summaries)
|
||||
|
|
@ -51,11 +59,7 @@ func (srv *AggregationService) Schedule() {
|
|||
go srv.persistWorker(summaries)
|
||||
}
|
||||
|
||||
// Run once initially
|
||||
srv.trigger(jobs)
|
||||
|
||||
gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.trigger, jobs)
|
||||
<-gocron.Start()
|
||||
return srv.trigger(jobs, userIds)
|
||||
}
|
||||
|
||||
func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) {
|
||||
|
|
@ -77,13 +81,22 @@ func (srv *AggregationService) persistWorker(summaries <-chan *models.Summary) {
|
|||
}
|
||||
}
|
||||
|
||||
func (srv *AggregationService) trigger(jobs chan<- *AggregationJob) error {
|
||||
func (srv *AggregationService) trigger(jobs chan<- *AggregationJob, userIds map[string]bool) error {
|
||||
log.Println("Generating summaries.")
|
||||
|
||||
users, err := srv.userService.GetAll()
|
||||
if err != nil {
|
||||
var users []*models.User
|
||||
if allUsers, err := srv.userService.GetAll(); err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
} else if userIds != nil && len(userIds) > 0 {
|
||||
users = make([]*models.User, len(userIds))
|
||||
for i, u := range allUsers {
|
||||
if yes, ok := userIds[u.ID]; yes && ok {
|
||||
users[i] = u
|
||||
}
|
||||
}
|
||||
} else {
|
||||
users = allUsers
|
||||
}
|
||||
|
||||
latestSummaries, err := srv.summaryService.GetLatestByUser()
|
||||
|
|
|
|||
|
|
@ -225,6 +225,10 @@ func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) {
|
|||
return srv.repository.GetLatestByUser()
|
||||
}
|
||||
|
||||
func (srv *SummaryService) DeleteByUser(userId string) error {
|
||||
return srv.repository.DeleteByUser(userId)
|
||||
}
|
||||
|
||||
func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, user *models.User, c chan models.SummaryItemContainer) {
|
||||
durations := make(map[string]time.Duration)
|
||||
|
||||
|
|
|
|||
|
|
@ -25,4 +25,4 @@ func ParseUserAgent(ua string) (string, string, error) {
|
|||
return "", "", errors.New("failed to parse user agent string")
|
||||
}
|
||||
return groups[0][1], groups[0][2], nil
|
||||
}
|
||||
}
|
||||
|
|
@ -1 +1 @@
|
|||
1.14.1
|
||||
1.15.0
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@
|
|||
</form>
|
||||
</div>
|
||||
|
||||
<div class="w-full mt-4 mb-8 pb-8">
|
||||
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
|
||||
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
|
||||
Reset API Key
|
||||
</div>
|
||||
|
|
@ -66,7 +66,7 @@
|
|||
</form>
|
||||
</div>
|
||||
|
||||
<div class="w-full mt-4 mb-8 pb-8">
|
||||
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
|
||||
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
|
||||
Language Mappings
|
||||
</div>
|
||||
|
|
@ -119,7 +119,7 @@
|
|||
|
||||
</div>
|
||||
|
||||
<div class="w-full mt-4 mb-8 pb-8">
|
||||
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
|
||||
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
|
||||
Badges
|
||||
</div>
|
||||
|
|
@ -169,6 +169,34 @@
|
|||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="w-full mt-4 mb-8 pb-8">
|
||||
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
|
||||
⚠️ Danger Zone
|
||||
</div>
|
||||
<div class="mt-10 text-gray-300 text-sm">
|
||||
<h3 class="font-semibold text-md mb-4 border-b border-green-700 inline-block">
|
||||
Regenerate summaries
|
||||
</h3>
|
||||
<p>
|
||||
Wakapi improves its efficiency and speed by automatically aggregating individual heartbeats to summaries on a per-day basis.
|
||||
That is, historic summaries, i.e. such from past days, are generated once and only fetched from the database in a static fashion afterwards, unless you pass <span class="font-mono font-normal bg-gray-900 p-1 rounded whitespace-no-wrap">&recompute=true</span> with your request.
|
||||
</p>
|
||||
<p class="mt-2">
|
||||
If, for some reason, these aggregated summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively), you may want to re-generate them from raw heartbeats.
|
||||
</p>
|
||||
<p class="mt-2">
|
||||
<strong>Note:</strong> Only run this action if you know what you are doing. Data might be lost is case heartbeats were deleted after the respective summaries had been generated.
|
||||
</p>
|
||||
</div>
|
||||
<div class="mt-10 flex justify-center">
|
||||
<form action="settings/regenerate" method="post" id="form-regenerate-summaries">
|
||||
<button type="button" class="py-1 px-3 rounded bg-red-500 hover:bg-red-600 text-white text-sm" id="btn-regenerate-summaries">
|
||||
Clear & Regenerate
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
|
||||
|
|
@ -182,6 +210,14 @@
|
|||
e.innerHTML = e.innerHTML.replace('%s', baseUrl)
|
||||
e.classList.remove('hidden')
|
||||
})
|
||||
|
||||
const btnRegenerate = document.querySelector("#btn-regenerate-summaries")
|
||||
const formRegenerate = document.querySelector('#form-regenerate-summaries')
|
||||
btnRegenerate.addEventListener('click', () => {
|
||||
if (confirm('Are you sure?')) {
|
||||
formRegenerate.submit()
|
||||
}
|
||||
})
|
||||
</script>
|
||||
|
||||
{{ template "footer.tpl.html" . }}
|
||||
|
|
|
|||
Loading…
Reference in New Issue