From beffe71ea60224e88f957505d35d31f2188d9a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Fri, 30 Sep 2022 17:14:05 +0200 Subject: [PATCH 01/16] feat: add leaderboard data model --- config/config.go | 3 +++ models/leaderboard.go | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 models/leaderboard.go diff --git a/config/config.go b/config/config.go index cff5a47..be82c86 100644 --- a/config/config.go +++ b/config/config.go @@ -216,6 +216,9 @@ func (c *Config) GetMigrationFunc(dbDialect string) models.MigrationFunc { if err := db.AutoMigrate(&models.Diagnostics{}); err != nil && !c.Db.AutoMigrateFailSilently { return err } + if err := db.AutoMigrate(&models.LeaderboardItem{}); err != nil && !c.Db.AutoMigrateFailSilently { + return err + } return nil } } diff --git a/models/leaderboard.go b/models/leaderboard.go new file mode 100644 index 0000000..32949f2 --- /dev/null +++ b/models/leaderboard.go @@ -0,0 +1,15 @@ +package models + +import "time" + +type LeaderboardItem struct { + ID uint `json:"-" gorm:"primary_key; size:32"` + User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` + UserID string `json:"user_id" gorm:"not null; index:idx_leaderboard_user"` + Rank uint `json:"rank" gorm:"not null; size:16"` + Interval string `json:"interval" gorm:"not null; size:32; index:idx_leaderboard_combined"` + By uint8 `json:"aggregated_by" gorm:"index:idx_leaderboard_combined"` + Total time.Duration `json:"total" gorm:"not null" swaggertype:"primitive,integer"` + Key string `json:"key" gorm:"size:255"` + CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"` +} From 13a3d9f03a20150bce1d45c0269c5e5d1e2296d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sun, 2 Oct 2022 00:01:39 +0200 Subject: [PATCH 02/16] feat: leaderboard generation and querying --- config/config.go | 21 ++--- main.go | 5 ++ models/leaderboard.go | 6 +- repositories/leaderboard.go | 63 +++++++++++++++ repositories/repositories.go | 8 ++ repositories/user.go | 8 ++ services/leaderboard.go | 149 +++++++++++++++++++++++++++++++++++ services/services.go | 10 +++ services/user.go | 4 + utils/db.go | 9 +++ 10 files changed, 270 insertions(+), 13 deletions(-) create mode 100644 repositories/leaderboard.go create mode 100644 services/leaderboard.go diff --git a/config/config.go b/config/config.go index be82c86..a15e6de 100644 --- a/config/config.go +++ b/config/config.go @@ -65,16 +65,17 @@ var cFlag = flag.String("config", defaultConfigPath, "config file location") var env string type appConfig struct { - AggregationTime string `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"` - ReportTimeWeekly string `yaml:"report_time_weekly" default:"fri,18:00" env:"WAKAPI_REPORT_TIME_WEEKLY"` - ImportBackoffMin int `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"` - ImportBatchSize int `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"` - InactiveDays int `yaml:"inactive_days" default:"7" env:"WAKAPI_INACTIVE_DAYS"` - HeartbeatMaxAge string `yaml:"heartbeat_max_age" default:"4320h" env:"WAKAPI_HEARTBEAT_MAX_AGE"` - CountCacheTTLMin int `yaml:"count_cache_ttl_min" default:"30" env:"WAKAPI_COUNT_CACHE_TTL_MIN"` - AvatarURLTemplate string `yaml:"avatar_url_template" default:"api/avatar/{username_hash}.svg" env:"WAKAPI_AVATAR_URL_TEMPLATE"` - CustomLanguages map[string]string `yaml:"custom_languages"` - Colors map[string]map[string]string `yaml:"-"` + AggregationTime string `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"` + LeaderboardGenerationTime string `yaml:"leaderboard_generation_time" default:"06:00,18:00" env:"WAKAPI_LEADERBOARD_GENERATION_TIME"` + ReportTimeWeekly string `yaml:"report_time_weekly" default:"fri,18:00" env:"WAKAPI_REPORT_TIME_WEEKLY"` + ImportBackoffMin int `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"` + ImportBatchSize int `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"` + InactiveDays int `yaml:"inactive_days" default:"7" env:"WAKAPI_INACTIVE_DAYS"` + HeartbeatMaxAge string `yaml:"heartbeat_max_age" default:"4320h" env:"WAKAPI_HEARTBEAT_MAX_AGE"` + CountCacheTTLMin int `yaml:"count_cache_ttl_min" default:"30" env:"WAKAPI_COUNT_CACHE_TTL_MIN"` + AvatarURLTemplate string `yaml:"avatar_url_template" default:"api/avatar/{username_hash}.svg" env:"WAKAPI_AVATAR_URL_TEMPLATE"` + CustomLanguages map[string]string `yaml:"custom_languages"` + Colors map[string]map[string]string `yaml:"-"` } type securityConfig struct { diff --git a/main.go b/main.go index 795c185..c289d36 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,7 @@ var ( languageMappingRepository repositories.ILanguageMappingRepository projectLabelRepository repositories.IProjectLabelRepository summaryRepository repositories.ISummaryRepository + leaderboardRepository *repositories.LeaderboardRepository keyValueRepository repositories.IKeyValueRepository diagnosticsRepository repositories.IDiagnosticsRepository metricsRepository *repositories.MetricsRepository @@ -72,6 +73,7 @@ var ( projectLabelService services.IProjectLabelService durationService services.IDurationService summaryService services.ISummaryService + leaderboardService services.ILeaderboardService aggregationService services.IAggregationService mailService services.IMailService keyValueService services.IKeyValueService @@ -159,6 +161,7 @@ func main() { languageMappingRepository = repositories.NewLanguageMappingRepository(db) projectLabelRepository = repositories.NewProjectLabelRepository(db) summaryRepository = repositories.NewSummaryRepository(db) + leaderboardRepository = repositories.NewLeaderboardRepository(db) keyValueRepository = repositories.NewKeyValueRepository(db) diagnosticsRepository = repositories.NewDiagnosticsRepository(db) metricsRepository = repositories.NewMetricsRepository(db) @@ -172,6 +175,7 @@ func main() { heartbeatService = services.NewHeartbeatService(heartbeatRepository, languageMappingService) durationService = services.NewDurationService(heartbeatService) summaryService = services.NewSummaryService(summaryRepository, durationService, aliasService, projectLabelService) + leaderboardService = services.NewLeaderboardService(leaderboardRepository, summaryService, userService) aggregationService = services.NewAggregationService(userService, summaryService, heartbeatService) keyValueService = services.NewKeyValueService(keyValueRepository) reportService = services.NewReportService(summaryService, userService, mailService) @@ -180,6 +184,7 @@ func main() { // Schedule background tasks go aggregationService.Schedule() + go leaderboardService.ScheduleDefault() go miscService.ScheduleCountTotalTime() go reportService.Schedule() diff --git a/models/leaderboard.go b/models/leaderboard.go index 32949f2..085347b 100644 --- a/models/leaderboard.go +++ b/models/leaderboard.go @@ -6,10 +6,10 @@ type LeaderboardItem struct { ID uint `json:"-" gorm:"primary_key; size:32"` User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"` UserID string `json:"user_id" gorm:"not null; index:idx_leaderboard_user"` - Rank uint `json:"rank" gorm:"not null; size:16"` + Rank uint `json:"rank" gorm:"->"` Interval string `json:"interval" gorm:"not null; size:32; index:idx_leaderboard_combined"` - By uint8 `json:"aggregated_by" gorm:"index:idx_leaderboard_combined"` + By *uint8 `json:"aggregated_by" gorm:"index:idx_leaderboard_combined"` // pointer because nullable Total time.Duration `json:"total" gorm:"not null" swaggertype:"primitive,integer"` - Key string `json:"key" gorm:"size:255"` + Key *string `json:"key" gorm:"size:255"` // pointer because nullable CreatedAt CustomTime `gorm:"type:timestamp; default:CURRENT_TIMESTAMP" swaggertype:"string" format:"date" example:"2006-01-02 15:04:05.000"` } diff --git a/repositories/leaderboard.go b/repositories/leaderboard.go new file mode 100644 index 0000000..366873a --- /dev/null +++ b/repositories/leaderboard.go @@ -0,0 +1,63 @@ +package repositories + +import ( + "github.com/muety/wakapi/models" + "github.com/muety/wakapi/utils" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +type LeaderboardRepository struct { + db *gorm.DB +} + +func NewLeaderboardRepository(db *gorm.DB) *LeaderboardRepository { + return &LeaderboardRepository{db: db} +} + +func (r *LeaderboardRepository) InsertBatch(items []*models.LeaderboardItem) error { + if err := r.db. + Clauses(clause.OnConflict{DoNothing: true}). + Create(&items).Error; err != nil { + return err + } + return nil +} + +func (r *LeaderboardRepository) GetAllAggregatedByInterval(key *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { + // TODO: distinct by (user, key) to filter out potential duplicates ? + var items []*models.LeaderboardItem + q := r.db. + Select("*, rank() over (partition by `key` order by total desc) as `rank`"). + Where("`interval` in ?", *key) + q = utils.WhereNullable(q, "`by`", by) + + if err := q.Find(&items).Error; err != nil { + return nil, err + } + return items, nil +} + +func (r *LeaderboardRepository) GetAggregatedByUserAndInterval(userId string, key *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { + var items []*models.LeaderboardItem + q := r.db. + Select("*, rank() over (partition by `key` order by total desc) as `rank`"). + Where("user_id = ?", userId). + Where("`interval` in ?", *key) + q = utils.WhereNullable(q, "`by`", by) + + if err := q.Find(&items).Error; err != nil { + return nil, err + } + return items, nil +} + +func (r *LeaderboardRepository) DeleteByUserAndInterval(userId string, key *models.IntervalKey) error { + if err := r.db. + Where("user_id = ?", userId). + Where("`interval` in ?", *key). + Delete(models.LeaderboardItem{}).Error; err != nil { + return err + } + return nil +} diff --git a/repositories/repositories.go b/repositories/repositories.go index 7a4af1f..e8361b2 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -76,6 +76,7 @@ type IUserRepository interface { GetByResetToken(string) (*models.User, error) GetAll() ([]*models.User, error) GetAllByReports(bool) ([]*models.User, error) + GetAllByLeaderboard(bool) ([]*models.User, error) GetByLoggedInAfter(time.Time) ([]*models.User, error) GetByLastActiveAfter(time.Time) ([]*models.User, error) Count() (int64, error) @@ -84,3 +85,10 @@ type IUserRepository interface { UpdateField(*models.User, string, interface{}) (*models.User, error) Delete(*models.User) error } + +type ILeaderboardRepository interface { + InsertBatch([]*models.LeaderboardItem) error + DeleteByUserAndInterval(string, *models.IntervalKey) error + GetAllAggregatedByInterval(*models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) + GetAggregatedByUserAndInterval(string, *models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) +} diff --git a/repositories/user.go b/repositories/user.go index 4b1a3d0..cca7b76 100644 --- a/repositories/user.go +++ b/repositories/user.go @@ -85,6 +85,14 @@ func (r *UserRepository) GetAllByReports(reportsEnabled bool) ([]*models.User, e return users, nil } +func (r *UserRepository) GetAllByLeaderboard(leaderboardEnabled bool) ([]*models.User, error) { + var users []*models.User + if err := r.db.Where(&models.User{PublicLeaderboard: leaderboardEnabled}).Find(&users).Error; err != nil { + return nil, err + } + return users, nil +} + func (r *UserRepository) GetByLoggedInAfter(t time.Time) ([]*models.User, error) { var users []*models.User if err := r.db. diff --git a/services/leaderboard.go b/services/leaderboard.go new file mode 100644 index 0000000..db015cd --- /dev/null +++ b/services/leaderboard.go @@ -0,0 +1,149 @@ +package services + +import ( + "github.com/emvi/logbuch" + "github.com/leandro-lugaresi/hub" + "github.com/muety/wakapi/config" + "github.com/muety/wakapi/models" + "github.com/muety/wakapi/repositories" + "github.com/muety/wakapi/utils" + "github.com/patrickmn/go-cache" + "time" +) + +type LeaderboardService struct { + config *config.Config + cache *cache.Cache + eventBus *hub.Hub + repository repositories.ILeaderboardRepository + summaryService ISummaryService + userService IUserService +} + +func NewLeaderboardService(leaderboardRepo repositories.ILeaderboardRepository, summaryService ISummaryService, userService IUserService) *LeaderboardService { + return &LeaderboardService{ + config: config.Get(), + cache: cache.New(24*time.Hour, 24*time.Hour), + eventBus: config.EventBus(), + repository: leaderboardRepo, + summaryService: summaryService, + userService: userService, + } +} + +func (srv *LeaderboardService) ScheduleDefault() { + runAllUsers := func(interval *models.IntervalKey, by []uint8) { + users, err := srv.userService.GetAllByLeaderboard(true) + if err != nil { + config.Log().Error("failed to get users for leaderboard generation - %v", err) + return + } + + srv.Run(users, interval, by) + } + + runAllUsers(models.IntervalPast7Days, []uint8{models.SummaryLanguage}) + + //s := gocron.NewScheduler(time.Local) + //s.Every(1).Day().At(srv.config.App.LeaderboardGenerationTime).Do(runAllUsers, models.IntervalPast7Days, []uint8{models.SummaryLanguage}) + //s.StartBlocking() +} + +func (srv *LeaderboardService) Run(users []*models.User, interval *models.IntervalKey, by []uint8) error { + logbuch.Info("generating leaderboard (%s) for %d users (%d aggregations)", (*interval)[0], len(users), len(by)) + + for _, user := range users { + if err := srv.repository.DeleteByUserAndInterval(user.ID, interval); err != nil { + config.Log().Error("failed to delete leaderboard items for user %s (interval %s) - %v", user.ID, (*interval)[0], err) + continue + } + + item, err := srv.GenerateByUser(user, interval) + if err != nil { + config.Log().Error("failed to generate general leaderboard for user %s - %v", user.ID, err) + continue + } + + if err := srv.repository.InsertBatch([]*models.LeaderboardItem{item}); err != nil { + config.Log().Error("failed to persist general leaderboard for user %s - %v", user.ID, err) + continue + } + + for _, by := range by { + items, err := srv.GenerateAggregatedByUser(user, interval, by) + if err != nil { + config.Log().Error("failed to generate aggregated (by %s) leaderboard for user %s - %v", models.GetEntityColumn(by), user.ID, err) + continue + } + + if len(items) == 0 { + continue + } + + if err := srv.repository.InsertBatch(items); err != nil { + config.Log().Error("failed to persist aggregated (by %s) leaderboard for user %s - %v", models.GetEntityColumn(by), user.ID, err) + continue + } + } + } + + logbuch.Info("finished leaderboard generation") + + return nil +} + +func (srv *LeaderboardService) GetByInterval(interval *models.IntervalKey) ([]*models.LeaderboardItem, error) { + return srv.GetAggregatedByInterval(interval, nil) +} + +func (srv *LeaderboardService) GetAggregatedByInterval(interval *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { + return srv.repository.GetAllAggregatedByInterval(interval, by) +} + +func (srv *LeaderboardService) GenerateByUser(user *models.User, interval *models.IntervalKey) (*models.LeaderboardItem, error) { + err, from, to := utils.ResolveIntervalTZ(interval, user.TZ()) + if err != nil { + return nil, err + } + + summary, err := srv.summaryService.Aliased(from, to, user, srv.summaryService.Retrieve, nil, false) + if err != nil { + return nil, err + } + + return &models.LeaderboardItem{ + User: user, + UserID: user.ID, + Interval: (*interval)[0], + Total: summary.TotalTime(), + }, nil +} + +func (srv *LeaderboardService) GenerateAggregatedByUser(user *models.User, interval *models.IntervalKey, by uint8) ([]*models.LeaderboardItem, error) { + err, from, to := utils.ResolveIntervalTZ(interval, user.TZ()) + if err != nil { + return nil, err + } + + summary, err := srv.summaryService.Aliased(from, to, user, srv.summaryService.Retrieve, nil, false) + if err != nil { + return nil, err + } + + summaryItems := *summary.ItemsByType(by) + items := make([]*models.LeaderboardItem, summaryItems.Len()) + + for i := 0; i < summaryItems.Len(); i++ { + key := summaryItems[i].Key + items[i] = &models.LeaderboardItem{ + User: user, + UserID: user.ID, + Interval: (*interval)[0], + By: &by, + Total: summary.TotalTimeByKey(by, key), + Key: &key, + } + } + + return items, nil +} diff --git a/services/services.go b/services/services.go index b37ad44..75320a8 100644 --- a/services/services.go +++ b/services/services.go @@ -97,6 +97,15 @@ type IReportService interface { Run(*models.User, time.Duration) error } +type ILeaderboardService interface { + ScheduleDefault() + Run([]*models.User, *models.IntervalKey, []uint8) error + GetByInterval(*models.IntervalKey) ([]*models.LeaderboardItem, error) + GetAggregatedByInterval(*models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) + GenerateByUser(*models.User, *models.IntervalKey) (*models.LeaderboardItem, error) + GenerateAggregatedByUser(*models.User, *models.IntervalKey, uint8) ([]*models.LeaderboardItem, error) +} + type IUserService interface { GetUserById(string) (*models.User, error) GetUserByKey(string) (*models.User, error) @@ -104,6 +113,7 @@ type IUserService interface { GetUserByResetToken(string) (*models.User, error) GetAll() ([]*models.User, error) GetAllByReports(bool) ([]*models.User, error) + GetAllByLeaderboard(bool) ([]*models.User, error) GetActive(bool) ([]*models.User, error) Count() (int64, error) CreateOrGet(*models.Signup, bool) (*models.User, bool, error) diff --git a/services/user.go b/services/user.go index 3e1483b..c1ce616 100644 --- a/services/user.go +++ b/services/user.go @@ -100,6 +100,10 @@ func (srv *UserService) GetAllByReports(reportsEnabled bool) ([]*models.User, er return srv.repository.GetAllByReports(reportsEnabled) } +func (srv *UserService) GetAllByLeaderboard(leaderboardEnabled bool) ([]*models.User, error) { + return srv.repository.GetAllByLeaderboard(leaderboardEnabled) +} + func (srv *UserService) GetActive(exact bool) ([]*models.User, error) { minDate := time.Now().AddDate(0, 0, -1*srv.config.App.InactiveDays) if !exact { diff --git a/utils/db.go b/utils/db.go index e8469a4..beaa009 100644 --- a/utils/db.go +++ b/utils/db.go @@ -1,8 +1,10 @@ package utils import ( + "fmt" "github.com/emvi/logbuch" "gorm.io/gorm" + "reflect" ) func IsCleanDB(db *gorm.DB) bool { @@ -30,3 +32,10 @@ func HasConstraints(db *gorm.DB) bool { logbuch.Warn("HasForeignKeyConstraints is not yet implemented for dialect '%s'", db.Dialector.Name()) return false } + +func WhereNullable(query *gorm.DB, col string, val any) *gorm.DB { + if val == nil || reflect.ValueOf(val).IsNil() { + return query.Where(fmt.Sprintf("%s is null", col)) + } + return query.Where(fmt.Sprintf("%s = ?", col), val) +} From 4a22a19cb00fa742d87781fdecb9ae75342fc1ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sun, 2 Oct 2022 10:13:39 +0200 Subject: [PATCH 03/16] chore: generate leaderboard when enabled in user settings --- config/config.go | 2 +- models/user.go | 7 ++++--- repositories/leaderboard.go | 9 +++++++++ repositories/repositories.go | 1 + routes/settings.go | 1 + services/leaderboard.go | 38 ++++++++++++++++++++++++++++++------ services/services.go | 1 + 7 files changed, 49 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index a15e6de..dc3be48 100644 --- a/config/config.go +++ b/config/config.go @@ -66,7 +66,7 @@ var env string type appConfig struct { AggregationTime string `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"` - LeaderboardGenerationTime string `yaml:"leaderboard_generation_time" default:"06:00,18:00" env:"WAKAPI_LEADERBOARD_GENERATION_TIME"` + LeaderboardGenerationTime string `yaml:"leaderboard_generation_time" default:"06:00;18:00" env:"WAKAPI_LEADERBOARD_GENERATION_TIME"` ReportTimeWeekly string `yaml:"report_time_weekly" default:"fri,18:00" env:"WAKAPI_REPORT_TIME_WEEKLY"` ImportBackoffMin int `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"` ImportBatchSize int `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"` diff --git a/models/user.go b/models/user.go index 8207c3d..3dd2d0f 100644 --- a/models/user.go +++ b/models/user.go @@ -66,9 +66,10 @@ type CredentialsReset struct { } type UserDataUpdate struct { - Email string `schema:"email"` - Location string `schema:"location"` - ReportsWeekly bool `schema:"reports_weekly"` + Email string `schema:"email"` + Location string `schema:"location"` + ReportsWeekly bool `schema:"reports_weekly"` + PublicLeaderboard bool `schema:"public_leaderboard"` } type TimeByUser struct { diff --git a/repositories/leaderboard.go b/repositories/leaderboard.go index 366873a..d39c700 100644 --- a/repositories/leaderboard.go +++ b/repositories/leaderboard.go @@ -24,6 +24,15 @@ func (r *LeaderboardRepository) InsertBatch(items []*models.LeaderboardItem) err return nil } +func (r *LeaderboardRepository) CountAllByUser(userId string) (int64, error) { + var count int64 + err := r.db. + Table("leaderboard_items"). + Where("user_id = ?", userId). + Count(&count).Error + return count, err +} + func (r *LeaderboardRepository) GetAllAggregatedByInterval(key *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { // TODO: distinct by (user, key) to filter out potential duplicates ? var items []*models.LeaderboardItem diff --git a/repositories/repositories.go b/repositories/repositories.go index e8361b2..b18b639 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -88,6 +88,7 @@ type IUserRepository interface { type ILeaderboardRepository interface { InsertBatch([]*models.LeaderboardItem) error + CountAllByUser(string) (int64, error) DeleteByUserAndInterval(string, *models.IntervalKey) error GetAllAggregatedByInterval(*models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) GetAggregatedByUserAndInterval(string, *models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) diff --git a/routes/settings.go b/routes/settings.go index 960ec9e..7ea56b8 100644 --- a/routes/settings.go +++ b/routes/settings.go @@ -182,6 +182,7 @@ func (h *SettingsHandler) actionUpdateUser(w http.ResponseWriter, r *http.Reques user.Email = payload.Email user.Location = payload.Location user.ReportsWeekly = payload.ReportsWeekly + user.PublicLeaderboard = payload.PublicLeaderboard if _, err := h.userSrvc.Update(user); err != nil { return http.StatusInternalServerError, "", conf.ErrInternalServerError diff --git a/services/leaderboard.go b/services/leaderboard.go index db015cd..29f0508 100644 --- a/services/leaderboard.go +++ b/services/leaderboard.go @@ -2,6 +2,7 @@ package services import ( "github.com/emvi/logbuch" + "github.com/go-co-op/gocron" "github.com/leandro-lugaresi/hub" "github.com/muety/wakapi/config" "github.com/muety/wakapi/models" @@ -21,7 +22,7 @@ type LeaderboardService struct { } func NewLeaderboardService(leaderboardRepo repositories.ILeaderboardRepository, summaryService ISummaryService, userService IUserService) *LeaderboardService { - return &LeaderboardService{ + srv := &LeaderboardService{ config: config.Get(), cache: cache.New(24*time.Hour, 24*time.Hour), eventBus: config.EventBus(), @@ -29,6 +30,28 @@ func NewLeaderboardService(leaderboardRepo repositories.ILeaderboardRepository, summaryService: summaryService, userService: userService, } + + onUserUpdate := srv.eventBus.Subscribe(0, config.EventUserUpdate) + go func(sub *hub.Subscription) { + for m := range sub.Receiver { + + // generate leaderboard for updated user, if leaderboard enabled and none present, yet + user := m.Fields[config.FieldPayload].(*models.User) + if user.PublicLeaderboard { + exists, err := srv.ExistsAnyByUser(user.ID) + if err != nil { + config.Log().Error("failed to check existing leaderboards upon user update - %v", err) + } + if !exists { + logbuch.Info("generating leaderboard for '%s' after settings update", user.ID) + srv.Run([]*models.User{user}, models.IntervalPast7Days, []uint8{models.SummaryLanguage}) + } + } + + } + }(&onUserUpdate) + + return srv } func (srv *LeaderboardService) ScheduleDefault() { @@ -42,11 +65,9 @@ func (srv *LeaderboardService) ScheduleDefault() { srv.Run(users, interval, by) } - runAllUsers(models.IntervalPast7Days, []uint8{models.SummaryLanguage}) - - //s := gocron.NewScheduler(time.Local) - //s.Every(1).Day().At(srv.config.App.LeaderboardGenerationTime).Do(runAllUsers, models.IntervalPast7Days, []uint8{models.SummaryLanguage}) - //s.StartBlocking() + s := gocron.NewScheduler(time.Local) + s.Every(1).Day().At(srv.config.App.LeaderboardGenerationTime).Do(runAllUsers, models.IntervalPast7Days, []uint8{models.SummaryLanguage}) + s.StartBlocking() } func (srv *LeaderboardService) Run(users []*models.User, interval *models.IntervalKey, by []uint8) error { @@ -92,6 +113,11 @@ func (srv *LeaderboardService) Run(users []*models.User, interval *models.Interv return nil } +func (srv *LeaderboardService) ExistsAnyByUser(userId string) (bool, error) { + count, err := srv.repository.CountAllByUser(userId) + return count > 0, err +} + func (srv *LeaderboardService) GetByInterval(interval *models.IntervalKey) ([]*models.LeaderboardItem, error) { return srv.GetAggregatedByInterval(interval, nil) } diff --git a/services/services.go b/services/services.go index 75320a8..4dc0283 100644 --- a/services/services.go +++ b/services/services.go @@ -100,6 +100,7 @@ type IReportService interface { type ILeaderboardService interface { ScheduleDefault() Run([]*models.User, *models.IntervalKey, []uint8) error + ExistsAnyByUser(string) (bool, error) GetByInterval(*models.IntervalKey) ([]*models.LeaderboardItem, error) GetAggregatedByInterval(*models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) GenerateByUser(*models.User, *models.IntervalKey) (*models.LeaderboardItem, error) From dba4da8641af4e35f33628caf0de239525a962d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sun, 2 Oct 2022 10:31:01 +0200 Subject: [PATCH 04/16] chore: caching for leaderboard --- services/leaderboard.go | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/services/leaderboard.go b/services/leaderboard.go index 29f0508..f4cc6ac 100644 --- a/services/leaderboard.go +++ b/services/leaderboard.go @@ -9,6 +9,8 @@ import ( "github.com/muety/wakapi/repositories" "github.com/muety/wakapi/utils" "github.com/patrickmn/go-cache" + "reflect" + "strings" "time" ) @@ -24,7 +26,7 @@ type LeaderboardService struct { func NewLeaderboardService(leaderboardRepo repositories.ILeaderboardRepository, summaryService ISummaryService, userService IUserService) *LeaderboardService { srv := &LeaderboardService{ config: config.Get(), - cache: cache.New(24*time.Hour, 24*time.Hour), + cache: cache.New(6*time.Hour, 6*time.Hour), eventBus: config.EventBus(), repository: leaderboardRepo, summaryService: summaryService, @@ -108,8 +110,8 @@ func (srv *LeaderboardService) Run(users []*models.User, interval *models.Interv } } + srv.cache.Flush() logbuch.Info("finished leaderboard generation") - return nil } @@ -123,7 +125,19 @@ func (srv *LeaderboardService) GetByInterval(interval *models.IntervalKey) ([]*m } func (srv *LeaderboardService) GetAggregatedByInterval(interval *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { - return srv.repository.GetAllAggregatedByInterval(interval, by) + // check cache + cacheKey := srv.getHash(interval, by) + if cacheResult, ok := srv.cache.Get(cacheKey); ok { + return cacheResult.([]*models.LeaderboardItem), nil + } + + items, err := srv.repository.GetAllAggregatedByInterval(interval, by) + if err != nil { + return nil, err + } + + srv.cache.SetDefault(cacheKey, items) + return items, nil } func (srv *LeaderboardService) GenerateByUser(user *models.User, interval *models.IntervalKey) (*models.LeaderboardItem, error) { @@ -173,3 +187,11 @@ func (srv *LeaderboardService) GenerateAggregatedByUser(user *models.User, inter return items, nil } + +func (srv *LeaderboardService) getHash(interval *models.IntervalKey, by *uint8) string { + k := strings.Join(*interval, "__") + if by != nil && !reflect.ValueOf(by).IsNil() { + k += "__" + models.GetEntityColumn(*by) + } + return k +} From b3fa032bdea62402acfaf263aad0f551b5721903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Mon, 3 Oct 2022 10:53:27 +0200 Subject: [PATCH 05/16] feat(wip): leaderboard ui --- config/templates.go | 1 + main.go | 2 + models/view/leaderboard.go | 22 +++++++ routes/leaderboard.go | 74 ++++++++++++++++++++++ static/assets/js/components/leaderboard.js | 14 ++++ views/index.tpl.html | 7 +- views/leaderboard.tpl.html | 58 +++++++++++++++++ views/login-btn.tpl.html | 10 +++ views/menu-main.tpl.html | 12 ++-- 9 files changed, 187 insertions(+), 13 deletions(-) create mode 100644 models/view/leaderboard.go create mode 100644 routes/leaderboard.go create mode 100644 static/assets/js/components/leaderboard.js create mode 100644 views/leaderboard.tpl.html create mode 100644 views/login-btn.tpl.html diff --git a/config/templates.go b/config/templates.go index 244d1d7..5eab0aa 100644 --- a/config/templates.go +++ b/config/templates.go @@ -9,4 +9,5 @@ const ( ResetPasswordTemplate = "reset-password.tpl.html" SettingsTemplate = "settings.tpl.html" SummaryTemplate = "summary.tpl.html" + LeaderboardTemplate = "leaderboard.tpl.html" ) diff --git a/main.go b/main.go index c289d36..28f0c53 100644 --- a/main.go +++ b/main.go @@ -212,6 +212,7 @@ func main() { // MVC Handlers summaryHandler := routes.NewSummaryHandler(summaryService, userService) settingsHandler := routes.NewSettingsHandler(userService, heartbeatService, summaryService, aliasService, aggregationService, languageMappingService, projectLabelService, keyValueService, mailService) + leaderboardHandler := routes.NewLeaderboardHandler(userService, leaderboardService) homeHandler := routes.NewHomeHandler(keyValueService) loginHandler := routes.NewLoginHandler(userService, mailService) imprintHandler := routes.NewImprintHandler(keyValueService) @@ -246,6 +247,7 @@ func main() { loginHandler.RegisterRoutes(rootRouter) imprintHandler.RegisterRoutes(rootRouter) summaryHandler.RegisterRoutes(rootRouter) + leaderboardHandler.RegisterRoutes(rootRouter) settingsHandler.RegisterRoutes(rootRouter) relayHandler.RegisterRoutes(rootRouter) diff --git a/models/view/leaderboard.go b/models/view/leaderboard.go new file mode 100644 index 0000000..ed8c03e --- /dev/null +++ b/models/view/leaderboard.go @@ -0,0 +1,22 @@ +package view + +import "github.com/muety/wakapi/models" + +type LeaderboardViewModel struct { + User *models.User + Items []*models.LeaderboardItem + ItemsByLanguage []*models.LeaderboardItem + ApiKey string + Success string + Error string +} + +func (s *LeaderboardViewModel) WithSuccess(m string) *LeaderboardViewModel { + s.Success = m + return s +} + +func (s *LeaderboardViewModel) WithError(m string) *LeaderboardViewModel { + s.Error = m + return s +} diff --git a/routes/leaderboard.go b/routes/leaderboard.go new file mode 100644 index 0000000..8905a86 --- /dev/null +++ b/routes/leaderboard.go @@ -0,0 +1,74 @@ +package routes + +import ( + "github.com/gorilla/mux" + conf "github.com/muety/wakapi/config" + "github.com/muety/wakapi/middlewares" + "github.com/muety/wakapi/models" + "github.com/muety/wakapi/models/view" + "github.com/muety/wakapi/services" + "net/http" +) + +type LeaderboardHandler struct { + config *conf.Config + userService services.IUserService + leaderboardService services.ILeaderboardService +} + +func NewLeaderboardHandler(userService services.IUserService, leaderboardService services.ILeaderboardService) *LeaderboardHandler { + return &LeaderboardHandler{ + config: conf.Get(), + userService: userService, + leaderboardService: leaderboardService, + } +} + +func (h *LeaderboardHandler) RegisterRoutes(router *mux.Router) { + r := router.PathPrefix("/leaderboard").Subrouter() + r.Use( + middlewares.NewAuthenticateMiddleware(h.userService). + WithRedirectTarget(defaultErrorRedirectTarget()). + WithOptionalFor([]string{"/"}). + Handler, + ) + r.Methods(http.MethodGet).HandlerFunc(h.GetIndex) +} + +func (h *LeaderboardHandler) GetIndex(w http.ResponseWriter, r *http.Request) { + if h.config.IsDev() { + loadTemplates() + } + templates[conf.LeaderboardTemplate].Execute(w, h.buildViewModel(r)) +} + +func (h *LeaderboardHandler) buildViewModel(r *http.Request) *view.LeaderboardViewModel { + user := middlewares.GetPrincipal(r) + + itemsGeneral, err := h.leaderboardService.GetByInterval(models.IntervalPast7Days) + if err != nil { + conf.Log().Request(r).Error("error while fetching general leaderboard items - %v", err) + return &view.LeaderboardViewModel{Error: criticalError} + } + + by := models.SummaryLanguage + itemsByLanguage, err := h.leaderboardService.GetAggregatedByInterval(models.IntervalPast7Days, &by) + if err != nil { + conf.Log().Request(r).Error("error while fetching general leaderboard items - %v", err) + return &view.LeaderboardViewModel{Error: criticalError} + } + + var apiKey string + if user != nil { + apiKey = user.ApiKey + } + + return &view.LeaderboardViewModel{ + User: user, + Items: itemsGeneral, + ItemsByLanguage: itemsByLanguage, + ApiKey: apiKey, + Success: r.URL.Query().Get("success"), + Error: r.URL.Query().Get("error"), + } +} diff --git a/static/assets/js/components/leaderboard.js b/static/assets/js/components/leaderboard.js new file mode 100644 index 0000000..8b6288e --- /dev/null +++ b/static/assets/js/components/leaderboard.js @@ -0,0 +1,14 @@ +PetiteVue.createApp({ + //$delimiters: ['${', '}'], // https://github.com/vuejs/petite-vue/pull/100 + activeTab: defaultTab, + isActive(tab) { + return this.activeTab === tab + }, + updateTab() { + this.activeTab = window.location.hash.slice(1) || defaultTab + }, + mounted() { + this.updateTab() + window.addEventListener('hashchange', () => this.updateTab()) + } +}).mount('#leaderboard-page') diff --git a/views/index.tpl.html b/views/index.tpl.html index 760b6f2..f03e0fe 100644 --- a/views/index.tpl.html +++ b/views/index.tpl.html @@ -9,12 +9,7 @@ {{ template "alerts.tpl.html" . }} -
- -
+{{ template "login-btn.tpl.html" . }}
diff --git a/views/leaderboard.tpl.html b/views/leaderboard.tpl.html new file mode 100644 index 0000000..3b6bd69 --- /dev/null +++ b/views/leaderboard.tpl.html @@ -0,0 +1,58 @@ + + + +{{ template "head.tpl.html" . }} + + + + + + +{{ template "alerts.tpl.html" . }} + +{{ if .User }} +{{ template "menu-main.tpl.html" . }} +{{ else }} +{{ template "header.tpl.html" . }} +{{ template "login-btn.tpl.html" . }} +{{ end }} + +
+
+

Leaderboard

+ + + +
+
    + {{ range $i, $item := .Items }} +
  1. {{ $item.Rank }} - {{ $item.UserID }} - {{ $item.Total | duration }}
  2. + {{ end }} +
+
+ +
+
    + {{ range $i, $item := .ItemsByLanguage }} +
  1. {{ $item.Rank }} - {{ $item.UserID }} - {{ $item.Total | duration }}
  2. + {{ end }} +
+
+
+
+ +{{ template "footer.tpl.html" . }} + +{{ template "foot.tpl.html" . }} + + + diff --git a/views/login-btn.tpl.html b/views/login-btn.tpl.html new file mode 100644 index 0000000..0781a56 --- /dev/null +++ b/views/login-btn.tpl.html @@ -0,0 +1,10 @@ + \ No newline at end of file diff --git a/views/menu-main.tpl.html b/views/menu-main.tpl.html index fa41b0e..87e3cd9 100644 --- a/views/menu-main.tpl.html +++ b/views/menu-main.tpl.html @@ -10,6 +10,11 @@ + + + + + - -