From 8a94fef06b1b975419c49f9d93a42a639b111951 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Fri, 23 Dec 2022 13:41:32 +0100 Subject: [PATCH] feat: implement computation of users first heartbeats data time --- config/config.go | 1 + main.go | 4 +- models/view/settings.go | 6 ++- repositories/key_value.go | 11 +++++ repositories/repositories.go | 1 + routes/settings.go | 8 +++ services/key_value.go | 4 ++ services/misc.go | 95 ++++++++++++++++++++++++++++++------ services/services.go | 3 +- views/settings.tpl.html | 4 +- 10 files changed, 117 insertions(+), 20 deletions(-) diff --git a/config/config.go b/config/config.go index 616b8e3..df56dd9 100644 --- a/config/config.go +++ b/config/config.go @@ -33,6 +33,7 @@ const ( KeyLatestTotalTime = "latest_total_time" KeyLatestTotalUsers = "latest_total_users" KeyLastImportImport = "last_import" + KeyFirstHeartbeat = "first_heartbeat" KeyNewsbox = "newsbox" SimpleDateFormat = "2006-01-02" diff --git a/main.go b/main.go index 468bd8d..d6132ac 100644 --- a/main.go +++ b/main.go @@ -183,14 +183,14 @@ func main() { reportService = services.NewReportService(summaryService, userService, mailService) diagnosticsService = services.NewDiagnosticsService(diagnosticsRepository) housekeepingService = services.NewHousekeepingService(userService, heartbeatService, summaryService) - miscService = services.NewMiscService(userService, summaryService, keyValueService) + miscService = services.NewMiscService(userService, heartbeatService, summaryService, keyValueService) // Schedule background tasks go aggregationService.Schedule() go leaderboardService.Schedule() go reportService.Schedule() go housekeepingService.Schedule() - go miscService.ScheduleCountTotalTime() + go miscService.Schedule() routes.Init() diff --git a/models/view/settings.go b/models/view/settings.go index d0430b2..3388f4c 100644 --- a/models/view/settings.go +++ b/models/view/settings.go @@ -1,6 +1,9 @@ package view -import "github.com/muety/wakapi/models" +import ( + "github.com/muety/wakapi/models" + "time" +) type SettingsViewModel struct { User *models.User @@ -10,6 +13,7 @@ type SettingsViewModel struct { Projects []string SubscriptionPrice string DataRetentionMonths int + UserFirstData time.Time SupportContact string ApiKey string Success string diff --git a/repositories/key_value.go b/repositories/key_value.go index 3e2ae31..c06440e 100644 --- a/repositories/key_value.go +++ b/repositories/key_value.go @@ -34,6 +34,17 @@ func (r *KeyValueRepository) GetString(key string) (*models.KeyStringValue, erro return kv, nil } +func (r *KeyValueRepository) Search(like string) ([]*models.KeyStringValue, error) { + var keyValues []*models.KeyStringValue + if err := r.db.Table("key_string_values"). + Where("`key` like ?", like). + Find(&keyValues). + Error; err != nil { + return nil, err + } + return keyValues, nil +} + func (r *KeyValueRepository) PutString(kv *models.KeyStringValue) error { result := r.db. Clauses(clause.OnConflict{ diff --git a/repositories/repositories.go b/repositories/repositories.go index 31ffbd9..8b36eb7 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -43,6 +43,7 @@ type IKeyValueRepository interface { GetString(string) (*models.KeyStringValue, error) PutString(*models.KeyStringValue) error DeleteString(string) error + Search(string) ([]*models.KeyStringValue, error) } type ILanguageMappingRepository interface { diff --git a/routes/settings.go b/routes/settings.go index 82c8ef1..74e2c37 100644 --- a/routes/settings.go +++ b/routes/settings.go @@ -739,6 +739,13 @@ func (h *SettingsHandler) buildViewModel(r *http.Request) *view.SettingsViewMode subscriptionPrice = h.config.Subscriptions.StandardPrice } + // user first data + var firstData time.Time + firstDataKv := h.keyValueSrvc.MustGetString(fmt.Sprintf("%s_%s", conf.KeyFirstHeartbeat, user.ID)) + if firstDataKv.Value != "" { + firstData, _ = time.Parse(time.RFC822Z, firstDataKv.Value) + } + return &view.SettingsViewModel{ User: user, LanguageMappings: mappings, @@ -746,6 +753,7 @@ func (h *SettingsHandler) buildViewModel(r *http.Request) *view.SettingsViewMode Labels: combinedLabels, Projects: projects, ApiKey: user.ApiKey, + UserFirstData: firstData, SubscriptionPrice: subscriptionPrice, SupportContact: h.config.App.SupportContact, DataRetentionMonths: h.config.App.DataRetentionMonths, diff --git a/services/key_value.go b/services/key_value.go index deef6bb..99fc039 100644 --- a/services/key_value.go +++ b/services/key_value.go @@ -22,6 +22,10 @@ func (srv *KeyValueService) GetString(key string) (*models.KeyStringValue, error return srv.repository.GetString(key) } +func (srv *KeyValueService) GetByPrefix(prefix string) ([]*models.KeyStringValue, error) { + return srv.repository.Search(prefix + "%") +} + func (srv *KeyValueService) MustGetString(key string) *models.KeyStringValue { kv, err := srv.repository.GetString(key) if err != nil { diff --git a/services/misc.go b/services/misc.go index e148a11..6060014 100644 --- a/services/misc.go +++ b/services/misc.go @@ -1,6 +1,7 @@ package services import ( + "fmt" "github.com/emvi/logbuch" "github.com/muety/artifex/v2" "github.com/muety/wakapi/config" @@ -14,36 +15,57 @@ import ( ) const ( - countUsersEvery = 1 * time.Hour + countUsersEvery = 1 * time.Hour + computeOldestDataEvery = 6 * time.Hour ) var countLock = sync.Mutex{} +var firstDataLock = sync.Mutex{} type MiscService struct { - config *config.Config - userService IUserService - summaryService ISummaryService - keyValueService IKeyValueService - queueDefault *artifex.Dispatcher - queueWorkers *artifex.Dispatcher + config *config.Config + userService IUserService + heartbeatService IHeartbeatService + summaryService ISummaryService + keyValueService IKeyValueService + queueDefault *artifex.Dispatcher + queueWorkers *artifex.Dispatcher } -func NewMiscService(userService IUserService, summaryService ISummaryService, keyValueService IKeyValueService) *MiscService { +func NewMiscService(userService IUserService, heartbeatService IHeartbeatService, summaryService ISummaryService, keyValueService IKeyValueService) *MiscService { return &MiscService{ - config: config.Get(), - userService: userService, - summaryService: summaryService, - keyValueService: keyValueService, - queueDefault: config.GetDefaultQueue(), - queueWorkers: config.GetQueue(config.QueueProcessing), + config: config.Get(), + userService: userService, + heartbeatService: heartbeatService, + summaryService: summaryService, + keyValueService: keyValueService, + queueDefault: config.GetDefaultQueue(), + queueWorkers: config.GetQueue(config.QueueProcessing), } } -func (srv *MiscService) ScheduleCountTotalTime() { +func (srv *MiscService) Schedule() { logbuch.Info("scheduling total time counting") if _, err := srv.queueDefault.DispatchEvery(srv.CountTotalTime, countUsersEvery); err != nil { config.Log().Error("failed to schedule user counting jobs, %v", err) } + + logbuch.Info("scheduling first data computing") + if _, err := srv.queueDefault.DispatchEvery(srv.ComputeOldestHeartbeats, computeOldestDataEvery); err != nil { + config.Log().Error("failed to schedule first data computing jobs, %v", err) + } + + // run once initially for a fresh instance + if !srv.existsUsersTotalTime() { + if err := srv.queueDefault.Dispatch(srv.CountTotalTime); err != nil { + config.Log().Error("failed to dispatch user counting jobs, %v", err) + } + } + if !srv.existsUsersFirstData() { + if err := srv.queueDefault.Dispatch(srv.ComputeOldestHeartbeats); err != nil { + config.Log().Error("failed to dispatch first data computing jobs, %v", err) + } + } } func (srv *MiscService) CountTotalTime() { @@ -95,6 +117,39 @@ func (srv *MiscService) CountTotalTime() { }(&pendingJobs) } +func (srv *MiscService) ComputeOldestHeartbeats() { + logbuch.Info("computing users' first data") + + if err := srv.queueWorkers.Dispatch(func() { + if ok := firstDataLock.TryLock(); !ok { + config.Log().Warn("couldn't acquire lock for computing users' first data, job is still pending") + return + } + defer firstDataLock.Unlock() + + results, err := srv.heartbeatService.GetFirstByUsers() + if err != nil { + config.Log().Error("failed to compute users' first data, %v", err) + } + + for _, entry := range results { + if entry.Time.T().IsZero() { + continue + } + + kvKey := fmt.Sprintf("%s_%s", config.KeyFirstHeartbeat, entry.User) + if err := srv.keyValueService.PutString(&models.KeyStringValue{ + Key: kvKey, + Value: entry.Time.T().Format(time.RFC822Z), + }); err != nil { + config.Log().Error("failed to save user's first heartbeat time: %v", err) + } + } + }); err != nil { + config.Log().Error("failed to enqueue computing first data for user, %v", err) + } +} + func (srv *MiscService) countUserTotalTime(userId string) time.Duration { result, err := srv.summaryService.Aliased(time.Time{}, time.Now(), &models.User{ID: userId}, srv.summaryService.Retrieve, nil, false) if err != nil { @@ -103,3 +158,13 @@ func (srv *MiscService) countUserTotalTime(userId string) time.Duration { } return result.TotalTime() } + +func (srv *MiscService) existsUsersTotalTime() bool { + results, _ := srv.keyValueService.GetByPrefix(config.KeyLatestTotalTime) + return len(results) > 0 +} + +func (srv *MiscService) existsUsersFirstData() bool { + results, _ := srv.keyValueService.GetByPrefix(config.KeyFirstHeartbeat) + return len(results) > 0 +} diff --git a/services/services.go b/services/services.go index dc4a653..d1d6908 100644 --- a/services/services.go +++ b/services/services.go @@ -12,7 +12,7 @@ type IAggregationService interface { } type IMiscService interface { - ScheduleCountTotalTime() + Schedule() CountTotalTime() } @@ -52,6 +52,7 @@ type IDiagnosticsService interface { type IKeyValueService interface { GetString(string) (*models.KeyStringValue, error) MustGetString(string) *models.KeyStringValue + GetByPrefix(string) ([]*models.KeyStringValue, error) PutString(*models.KeyStringValue) error DeleteString(string) error } diff --git a/views/settings.tpl.html b/views/settings.tpl.html index 8c09024..6f3102b 100644 --- a/views/settings.tpl.html +++ b/views/settings.tpl.html @@ -701,10 +701,12 @@
{{ end }} + {{ if not .UserFirstData.IsZero }} - Your currently oldest data point is from 2022-01-01. + Your currently oldest data point is from {{ .UserFirstData | datetime }}.
+ {{ end }} Subscription status: