From 41f6db8f3459efe2db5192a16d58b788e2ba3032 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sun, 16 Oct 2022 18:59:19 +0200 Subject: [PATCH] feat(wip): leaderboard pagination (resolve #417) [ci-skip] --- models/shared.go | 19 +++++++++++++++++++ models/view/leaderboard.go | 1 + repositories/leaderboard.go | 30 ++++++++++++++++++++++++------ repositories/repositories.go | 4 ++-- routes/leaderboard.go | 7 +++++-- services/leaderboard.go | 14 ++++++++------ services/services.go | 4 ++-- utils/db.go | 10 ++++++++++ utils/http.go | 14 ++++++++++++++ 9 files changed, 85 insertions(+), 18 deletions(-) diff --git a/models/shared.go b/models/shared.go index a38c30d..a7054d4 100644 --- a/models/shared.go +++ b/models/shared.go @@ -34,6 +34,11 @@ type KeyedInterval struct { Key *IntervalKey } +type PageParams struct { + Page int `json:"page"` + PageSize int `json:"page_size"` +} + // CustomTime is a wrapper type around time.Time, mainly used for the purpose of transparently unmarshalling Python timestamps in the format . (e.g. 1619335137.3324468) type CustomTime time.Time @@ -99,3 +104,17 @@ func (j CustomTime) T() time.Time { func (j CustomTime) Valid() bool { return j.T().Unix() >= 0 } + +func (p *PageParams) Limit() int { + if p.PageSize < 0 { + return 0 + } + return p.PageSize +} + +func (p *PageParams) Offset() int { + if p.PageSize <= 0 { + return 0 + } + return (p.Page - 1) * p.PageSize +} diff --git a/models/view/leaderboard.go b/models/view/leaderboard.go index b564cc0..c701874 100644 --- a/models/view/leaderboard.go +++ b/models/view/leaderboard.go @@ -14,6 +14,7 @@ type LeaderboardViewModel struct { TopKeys []string UserLanguages map[string][]string ApiKey string + PageParams *models.PageParams Success string Error string } diff --git a/repositories/leaderboard.go b/repositories/leaderboard.go index 237194f..4bc819c 100644 --- a/repositories/leaderboard.go +++ b/repositories/leaderboard.go @@ -33,13 +33,17 @@ func (r *LeaderboardRepository) CountAllByUser(userId string) (int64, error) { return count, err } -func (r *LeaderboardRepository) GetAllAggregatedByInterval(key *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { +func (r *LeaderboardRepository) GetAllAggregatedByInterval(key *models.IntervalKey, by *uint8, limit, skip int) ([]*models.LeaderboardItem, error) { // TODO: distinct by (user, key) to filter out potential duplicates ? var items []*models.LeaderboardItem - q := r.db. + subq := r.db. + Table("leaderboard_items"). Select("*, rank() over (partition by \"key\" order by total desc) as \"rank\""). Where("\"interval\" in ?", *key) - q = utils.WhereNullable(q, "\"by\"", by) + subq = utils.WhereNullable(subq, "\"by\"", by) + + q := r.db.Table("(?) as ranked", subq) + q = r.withPaging(q, limit, skip) if err := q.Find(&items).Error; err != nil { return nil, err @@ -47,13 +51,17 @@ func (r *LeaderboardRepository) GetAllAggregatedByInterval(key *models.IntervalK return items, nil } -func (r *LeaderboardRepository) GetAggregatedByUserAndInterval(userId string, key *models.IntervalKey, by *uint8) ([]*models.LeaderboardItem, error) { +func (r *LeaderboardRepository) GetAggregatedByUserAndInterval(userId string, key *models.IntervalKey, by *uint8, limit, skip int) ([]*models.LeaderboardItem, error) { var items []*models.LeaderboardItem - q := r.db. + subq := r.db. + Table("leaderboard_items"). Select("*, rank() over (partition by \"key\" order by total desc) as \"rank\""). Where("user_id = ?", userId). Where("\"interval\" in ?", *key) - q = utils.WhereNullable(q, "\"by\"", by) + subq = utils.WhereNullable(subq, "\"by\"", by) + + q := r.db.Table("(?) as ranked", subq) + q = r.withPaging(q, limit, skip) if err := q.Find(&items).Error; err != nil { return nil, err @@ -79,3 +87,13 @@ func (r *LeaderboardRepository) DeleteByUserAndInterval(userId string, key *mode } return nil } + +func (r *LeaderboardRepository) withPaging(q *gorm.DB, limit, skip int) *gorm.DB { + if limit > 0 { + q = q.Where("\"rank\" <= ?", skip+limit) + } + if skip > 0 { + q = q.Where("\"rank\" > ?", skip) + } + return q +} diff --git a/repositories/repositories.go b/repositories/repositories.go index a78639e..1ce9b3b 100644 --- a/repositories/repositories.go +++ b/repositories/repositories.go @@ -92,6 +92,6 @@ type ILeaderboardRepository interface { CountAllByUser(string) (int64, error) DeleteByUser(string) error DeleteByUserAndInterval(string, *models.IntervalKey) error - GetAllAggregatedByInterval(*models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) - GetAggregatedByUserAndInterval(string, *models.IntervalKey, *uint8) ([]*models.LeaderboardItem, error) + GetAllAggregatedByInterval(*models.IntervalKey, *uint8, int, int) ([]*models.LeaderboardItem, error) + GetAggregatedByUserAndInterval(string, *models.IntervalKey, *uint8, int, int) ([]*models.LeaderboardItem, error) } diff --git a/routes/leaderboard.go b/routes/leaderboard.go index 7e4c625..dd1305c 100644 --- a/routes/leaderboard.go +++ b/routes/leaderboard.go @@ -10,6 +10,7 @@ import ( "github.com/muety/wakapi/models" "github.com/muety/wakapi/models/view" "github.com/muety/wakapi/services" + "github.com/muety/wakapi/utils" "net/http" "strings" ) @@ -56,6 +57,7 @@ func (h *LeaderboardHandler) buildViewModel(r *http.Request) *view.LeaderboardVi user := middlewares.GetPrincipal(r) byParam := strings.ToLower(r.URL.Query().Get("by")) keyParam := strings.ToLower(r.URL.Query().Get("key")) + pageParams := utils.ParsePageParams(r) var err error var leaderboard models.Leaderboard @@ -63,14 +65,14 @@ func (h *LeaderboardHandler) buildViewModel(r *http.Request) *view.LeaderboardVi var topKeys []string if byParam == "" { - leaderboard, err = h.leaderboardService.GetByInterval(models.IntervalPast7Days, true) + leaderboard, err = h.leaderboardService.GetByInterval(models.IntervalPast7Days, pageParams, true) if err != nil { conf.Log().Request(r).Error("error while fetching general leaderboard items - %v", err) return &view.LeaderboardViewModel{Error: criticalError} } } else { if by, ok := allowedAggregations[byParam]; ok { - leaderboard, err = h.leaderboardService.GetAggregatedByInterval(models.IntervalPast7Days, &by, true) + leaderboard, err = h.leaderboardService.GetAggregatedByInterval(models.IntervalPast7Days, &by, pageParams, true) if err != nil { conf.Log().Request(r).Error("error while fetching general leaderboard items - %v", err) return &view.LeaderboardViewModel{Error: criticalError} @@ -110,6 +112,7 @@ func (h *LeaderboardHandler) buildViewModel(r *http.Request) *view.LeaderboardVi UserLanguages: userLanguages, TopKeys: topKeys, ApiKey: apiKey, + PageParams: pageParams, Success: r.URL.Query().Get("success"), Error: r.URL.Query().Get("error"), } diff --git a/services/leaderboard.go b/services/leaderboard.go index b47c01b..ab3bfcd 100644 --- a/services/leaderboard.go +++ b/services/leaderboard.go @@ -10,6 +10,7 @@ import ( "github.com/muety/wakapi/utils" "github.com/patrickmn/go-cache" "reflect" + "strconv" "strings" "time" ) @@ -125,18 +126,18 @@ func (srv *LeaderboardService) ExistsAnyByUser(userId string) (bool, error) { return count > 0, err } -func (srv *LeaderboardService) GetByInterval(interval *models.IntervalKey, resolveUsers bool) (models.Leaderboard, error) { - return srv.GetAggregatedByInterval(interval, nil, resolveUsers) +func (srv *LeaderboardService) GetByInterval(interval *models.IntervalKey, pageParams *models.PageParams, resolveUsers bool) (models.Leaderboard, error) { + return srv.GetAggregatedByInterval(interval, nil, pageParams, resolveUsers) } -func (srv *LeaderboardService) GetAggregatedByInterval(interval *models.IntervalKey, by *uint8, resolveUsers bool) (models.Leaderboard, error) { +func (srv *LeaderboardService) GetAggregatedByInterval(interval *models.IntervalKey, by *uint8, pageParams *models.PageParams, resolveUsers bool) (models.Leaderboard, error) { // check cache - cacheKey := srv.getHash(interval, by) + cacheKey := srv.getHash(interval, by, pageParams) if cacheResult, ok := srv.cache.Get(cacheKey); ok { return cacheResult.([]*models.LeaderboardItem), nil } - items, err := srv.repository.GetAllAggregatedByInterval(interval, by) + items, err := srv.repository.GetAllAggregatedByInterval(interval, by, pageParams.Limit(), pageParams.Offset()) if err != nil { return nil, err } @@ -208,10 +209,11 @@ func (srv *LeaderboardService) GenerateAggregatedByUser(user *models.User, inter return items, nil } -func (srv *LeaderboardService) getHash(interval *models.IntervalKey, by *uint8) string { +func (srv *LeaderboardService) getHash(interval *models.IntervalKey, by *uint8, pageParams *models.PageParams) string { k := strings.Join(*interval, "__") if by != nil && !reflect.ValueOf(by).IsNil() { k += "__" + models.GetEntityColumn(*by) } + k = "__" + strconv.Itoa(pageParams.Page) + "__" + strconv.Itoa(pageParams.PageSize) return k } diff --git a/services/services.go b/services/services.go index 06140c4..c41b539 100644 --- a/services/services.go +++ b/services/services.go @@ -101,8 +101,8 @@ type ILeaderboardService interface { ScheduleDefault() Run([]*models.User, *models.IntervalKey, []uint8) error ExistsAnyByUser(string) (bool, error) - GetByInterval(*models.IntervalKey, bool) (models.Leaderboard, error) - GetAggregatedByInterval(*models.IntervalKey, *uint8, bool) (models.Leaderboard, error) + GetByInterval(*models.IntervalKey, *models.PageParams, bool) (models.Leaderboard, error) + GetAggregatedByInterval(*models.IntervalKey, *uint8, *models.PageParams, bool) (models.Leaderboard, error) GenerateByUser(*models.User, *models.IntervalKey) (*models.LeaderboardItem, error) GenerateAggregatedByUser(*models.User, *models.IntervalKey, uint8) ([]*models.LeaderboardItem, error) } diff --git a/utils/db.go b/utils/db.go index beaa009..48e3d70 100644 --- a/utils/db.go +++ b/utils/db.go @@ -39,3 +39,13 @@ func WhereNullable(query *gorm.DB, col string, val any) *gorm.DB { } return query.Where(fmt.Sprintf("%s = ?", col), val) } + +func WithPaging(query *gorm.DB, limit, skip int) *gorm.DB { + if limit >= 0 { + query = query.Limit(limit) + } + if skip >= 0 { + query = query.Offset(skip) + } + return query +} diff --git a/utils/http.go b/utils/http.go index e2ab8b9..2edfad5 100644 --- a/utils/http.go +++ b/utils/http.go @@ -3,6 +3,7 @@ package utils import ( "encoding/json" "github.com/muety/wakapi/config" + "github.com/muety/wakapi/models" "net/http" "regexp" "strconv" @@ -42,3 +43,16 @@ func IsNoCache(r *http.Request, cacheTtl time.Duration) bool { } return false } + +func ParsePageParams(r *http.Request) *models.PageParams { + pageParams := &models.PageParams{} + page := r.URL.Query().Get("page") + pageSize := r.URL.Query().Get("page_size") + if p, err := strconv.Atoi(page); err == nil { + pageParams.Page = p + } + if p, err := strconv.Atoi(pageSize); err == nil && pageParams.Page > 0 { + pageParams.PageSize = p + } + return pageParams +}