From 2f12d8efdefa1515a716baa3f42c58dfa8cf7f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sat, 7 Nov 2020 12:01:35 +0100 Subject: [PATCH] refactor: simplify summary generation (resolve #68) --- models/heartbeat.go | 21 ++ models/heartbeats.go | 38 +++ models/shared.go | 9 + models/summary.go | 52 ++++ models/user.go | 5 + repositories/heartbeart.go | 20 +- repositories/summary.go | 19 +- routes/compat/shields/v1/badge.go | 9 +- routes/compat/wakatime/v1/all_time.go | 9 +- routes/compat/wakatime/v1/summaries.go | 4 +- routes/summary.go | 9 +- services/aggregation.go | 85 +++--- services/heartbeat.go | 4 +- services/summary.go | 353 ++++++++++--------------- version.txt | 2 +- 15 files changed, 352 insertions(+), 287 deletions(-) create mode 100644 models/heartbeats.go diff --git a/models/heartbeat.go b/models/heartbeat.go index e3be990..0eb05f1 100644 --- a/models/heartbeat.go +++ b/models/heartbeat.go @@ -41,3 +41,24 @@ func (h *Heartbeat) Augment(languageMappings map[string]string) { } h.Language, _ = languageMappings[ending] } + +func (h *Heartbeat) GetKey(t uint8) (key string) { + switch t { + case SummaryProject: + key = h.Project + case SummaryEditor: + key = h.Editor + case SummaryLanguage: + key = h.Language + case SummaryOS: + key = h.OperatingSystem + case SummaryMachine: + key = h.Machine + } + + if key == "" { + key = UnknownSummaryKey + } + + return key +} diff --git a/models/heartbeats.go b/models/heartbeats.go new file mode 100644 index 0000000..28e1d66 --- /dev/null +++ b/models/heartbeats.go @@ -0,0 +1,38 @@ +package models + +import "sort" + +type Heartbeats []*Heartbeat + +func (h Heartbeats) Len() int { + return len(h) +} + +func (h Heartbeats) Less(i, j int) bool { + return h[i].Time.T().Before(h[j].Time.T()) +} + +func (h Heartbeats) Swap(i, j int) { + h[i], h[j] = h[j], h[i] +} + +func (h *Heartbeats) Sorted() *Heartbeats { + sort.Sort(h) + return h +} + +func (h *Heartbeats) First() *Heartbeat { + // assumes slice to be sorted + if h.Len() == 0 { + return nil + } + return (*h)[0] +} + +func (h *Heartbeats) Last() *Heartbeat { + // assumes slice to be sorted + if h.Len() == 0 { + return nil + } + return (*h)[h.Len()-1] +} diff --git a/models/shared.go b/models/shared.go index dc35cb5..1612040 100644 --- a/models/shared.go +++ b/models/shared.go @@ -24,6 +24,11 @@ type KeyStringValue struct { Value string `gorm:"type:text"` } +type Interval struct { + Start time.Time + End time.Time +} + type CustomTime time.Time func (j *CustomTime) UnmarshalJSON(b []byte) error { @@ -79,3 +84,7 @@ func (j CustomTime) String() string { func (j CustomTime) T() time.Time { return time.Time(j) } + +func (j CustomTime) Valid() bool { + return j.T().Unix() >= 0 +} diff --git a/models/summary.go b/models/summary.go index fc44193..c5361b4 100644 --- a/models/summary.go +++ b/models/summary.go @@ -75,6 +75,8 @@ type SummaryParams struct { Recompute bool } +type AliasResolver func(t uint8, k string) string + func SummaryTypes() []uint8 { return []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine} } @@ -178,3 +180,53 @@ func (s *Summary) TotalTimeByFilters(filter *Filters) (timeSum time.Duration) { } return timeSum } + +func (s *Summary) WithResolvedAliases(resolve AliasResolver) *Summary { + processAliases := func(origin []*SummaryItem) []*SummaryItem { + target := make([]*SummaryItem, 0) + + findItem := func(key string) *SummaryItem { + for _, item := range target { + if item.Key == key { + return item + } + } + return nil + } + + for _, item := range origin { + // Add all "top-level" items, i.e. such without aliases + if key := resolve(item.Type, item.Key); key == item.Key { + target = append(target, item) + } + } + + for _, item := range origin { + // Add all remaining projects and merge with their alias + if key := resolve(item.Type, item.Key); key != item.Key { + if targetItem := findItem(key); targetItem != nil { + targetItem.Total += item.Total + } else { + target = append(target, &SummaryItem{ + ID: item.ID, + SummaryID: item.SummaryID, + Type: item.Type, + Key: key, + Total: item.Total, + }) + } + } + } + + return target + } + + // Resolve aliases + s.Projects = processAliases(s.Projects) + s.Editors = processAliases(s.Editors) + s.Languages = processAliases(s.Languages) + s.OperatingSystems = processAliases(s.OperatingSystems) + s.Machines = processAliases(s.Machines) + + return s +} diff --git a/models/user.go b/models/user.go index e99182a..ccca907 100644 --- a/models/user.go +++ b/models/user.go @@ -26,6 +26,11 @@ type CredentialsReset struct { PasswordRepeat string `schema:"password_repeat"` } +type TimeByUser struct { + User string + Time CustomTime +} + func (c *CredentialsReset) IsValid() bool { return validatePassword(c.PasswordNew) && c.PasswordNew == c.PasswordRepeat diff --git a/repositories/heartbeart.go b/repositories/heartbeart.go index 3fcd320..5052eba 100644 --- a/repositories/heartbeart.go +++ b/repositories/heartbeart.go @@ -34,18 +34,14 @@ func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User return heartbeats, nil } -// Will return *models.Heartbeat object with only user_id and time fields filled -func (r *HeartbeatRepository) GetFirstByUsers(userIds []string) ([]*models.Heartbeat, error) { - var heartbeats []*models.Heartbeat - if err := r.db. - Table("heartbeats"). - Select("user_id, min(time) as time"). - Where("user_id IN (?)", userIds). - Group("user_id"). - Scan(&heartbeats).Error; err != nil { - return nil, err - } - return heartbeats, nil +func (r *HeartbeatRepository) GetFirstByUsers() ([]*models.TimeByUser, error) { + var result []*models.TimeByUser + r.db.Model(&models.User{}). + Select("users.id as user, min(time) as time"). + Joins("left join heartbeats on users.id = heartbeats.user_id"). + Group("user"). + Scan(&result) + return result, nil } func (r *HeartbeatRepository) DeleteBefore(t time.Time) error { diff --git a/repositories/summary.go b/repositories/summary.go index 80f82d7..20f04a3 100644 --- a/repositories/summary.go +++ b/repositories/summary.go @@ -39,17 +39,14 @@ func (r *SummaryRepository) GetByUserWithin(user *models.User, from, to time.Tim return summaries, nil } -// Will return *models.Index objects with only user_id and to_time filled -func (r *SummaryRepository) GetLatestByUser() ([]*models.Summary, error) { - var summaries []*models.Summary - if err := r.db. - Table("summaries"). - Select("user_id, max(to_time) as to_time"). - Group("user_id"). - Scan(&summaries).Error; err != nil { - return nil, err - } - return summaries, nil +func (r *SummaryRepository) GetLastByUser() ([]*models.TimeByUser, error) { + var result []*models.TimeByUser + r.db.Model(&models.User{}). + Select("users.id as user, max(to_time) as time"). + Joins("left join summaries on users.id = summaries.user_id"). + Group("user"). + Scan(&result) + return result, nil } func (r *SummaryRepository) DeleteByUser(userId string) error { diff --git a/routes/compat/shields/v1/badge.go b/routes/compat/shields/v1/badge.go index 651012f..868e9c3 100644 --- a/routes/compat/shields/v1/badge.go +++ b/routes/compat/shields/v1/badge.go @@ -97,9 +97,12 @@ func (h *BadgeHandler) loadUserSummary(user *models.User, interval string) (*mod User: user, } - summary, err := h.summarySrvc.PostProcessWrapped( - h.summarySrvc.Construct(summaryParams.From, summaryParams.To, summaryParams.User, summaryParams.Recompute), - ) + var retrieveSummary services.SummaryRetriever = h.summarySrvc.Retrieve + if summaryParams.Recompute { + retrieveSummary = h.summarySrvc.Summarize + } + + summary, err := h.summarySrvc.Aliased(summaryParams.From, summaryParams.To, summaryParams.User, retrieveSummary) if err != nil { return nil, err, http.StatusInternalServerError } diff --git a/routes/compat/wakatime/v1/all_time.go b/routes/compat/wakatime/v1/all_time.go index 68ad6e8..c7e4433 100644 --- a/routes/compat/wakatime/v1/all_time.go +++ b/routes/compat/wakatime/v1/all_time.go @@ -55,9 +55,12 @@ func (h *AllTimeHandler) loadUserSummary(user *models.User) (*models.Summary, er Recompute: false, } - summary, err := h.summarySrvc.PostProcessWrapped( - h.summarySrvc.Construct(summaryParams.From, summaryParams.To, summaryParams.User, summaryParams.Recompute), // 'to' is always constant - ) + var retrieveSummary services.SummaryRetriever = h.summarySrvc.Retrieve + if summaryParams.Recompute { + retrieveSummary = h.summarySrvc.Summarize + } + + summary, err := h.summarySrvc.Aliased(summaryParams.From, summaryParams.To, summaryParams.User, retrieveSummary) if err != nil { return nil, err, http.StatusInternalServerError } diff --git a/routes/compat/wakatime/v1/summaries.go b/routes/compat/wakatime/v1/summaries.go index 4565377..77f5971 100644 --- a/routes/compat/wakatime/v1/summaries.go +++ b/routes/compat/wakatime/v1/summaries.go @@ -86,9 +86,7 @@ func (h *SummariesHandler) loadUserSummaries(r *http.Request) ([]*models.Summary summaries := make([]*models.Summary, len(intervals)) for i, interval := range intervals { - summary, err := h.summarySrvc.PostProcessWrapped( - h.summarySrvc.Construct(interval[0], interval[1], user, false), // 'to' is always constant - ) + summary, err := h.summarySrvc.Aliased(interval[0], interval[1], user, h.summarySrvc.Retrieve) if err != nil { return nil, err, http.StatusInternalServerError } diff --git a/routes/summary.go b/routes/summary.go index 3f1745d..0a8de3e 100644 --- a/routes/summary.go +++ b/routes/summary.go @@ -72,9 +72,12 @@ func (h *SummaryHandler) loadUserSummary(r *http.Request) (*models.Summary, erro return nil, err, http.StatusBadRequest } - summary, err := h.summarySrvc.PostProcessWrapped( - h.summarySrvc.Construct(summaryParams.From, summaryParams.To, summaryParams.User, summaryParams.Recompute), // 'to' is always constant - ) + var retrieveSummary services.SummaryRetriever = h.summarySrvc.Retrieve + if summaryParams.Recompute { + retrieveSummary = h.summarySrvc.Summarize + } + + summary, err := h.summarySrvc.Aliased(summaryParams.From, summaryParams.To, summaryParams.User, retrieveSummary) if err != nil { return nil, err, http.StatusInternalServerError } diff --git a/services/aggregation.go b/services/aggregation.go index 596bb3d..dc384c8 100644 --- a/services/aggregation.go +++ b/services/aggregation.go @@ -59,12 +59,19 @@ func (srv *AggregationService) Run(userIds map[string]bool) error { go srv.persistWorker(summaries) } + // don't leak open channels + go func(c1 chan *AggregationJob, c2 chan *models.Summary) { + defer close(c1) + defer close(c2) + time.Sleep(1 * time.Hour) + }(jobs, summaries) + return srv.trigger(jobs, userIds) } func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) { for job := range jobs { - if summary, err := srv.summaryService.Construct(job.From, job.To, &models.User{ID: job.UserID}, true); err != nil { + if summary, err := srv.summaryService.Summarize(job.From, job.To, &models.User{ID: job.UserID}); err != nil { log.Printf("Failed to generate summary (%v, %v, %s) – %v.\n", job.From, job.To, job.UserID, err) } else { log.Printf("Successfully generated summary (%v, %v, %s).\n", job.From, job.To, job.UserID) @@ -99,57 +106,59 @@ func (srv *AggregationService) trigger(jobs chan<- *AggregationJob, userIds map[ users = allUsers } - latestSummaries, err := srv.summaryService.GetLatestByUser() + // Get a map from user ids to the time of their latest summary or nil if none exists yet + lastUserSummaryTimes, err := srv.summaryService.GetLatestByUser() if err != nil { log.Println(err) return err } - userSummaryTimes := make(map[string]time.Time) - for _, s := range latestSummaries { - userSummaryTimes[s.UserID] = s.ToTime.T() + // Get a map from user ids to the time of their earliest heartbeats or nil if none exists yet + firstUserHeartbeatTimes, err := srv.heartbeatService.GetFirstByUsers() + if err != nil { + log.Println(err) + return err } - missingUserIDs := make([]string, 0) - for _, u := range users { - if _, ok := userSummaryTimes[u.ID]; !ok { - missingUserIDs = append(missingUserIDs, u.ID) + // Build actual lookup table from it + firstUserHeartbeatLookup := make(map[string]models.CustomTime) + for _, e := range firstUserHeartbeatTimes { + firstUserHeartbeatLookup[e.User] = e.Time + } + + // Generate summary aggregation jobs + for _, e := range lastUserSummaryTimes { + if e.Time.Valid() { + // Case 1: User has aggregated summaries already + // -> Spawn jobs to create summaries from their latest aggregation to now + generateUserJobs(e.User, e.Time.T(), jobs) + } else if t := firstUserHeartbeatLookup[e.User]; t.Valid() { + // Case 2: User has no aggregated summaries, yet, but has heartbeats + // -> Spawn jobs to create summaries from their first heartbeat to now + generateUserJobs(e.User, t.T(), jobs) } - } - - firstHeartbeats, err := srv.heartbeatService.GetFirstUserHeartbeats(missingUserIDs) - if err != nil { - log.Println(err) - return err - } - - for id, t := range userSummaryTimes { - generateUserJobs(id, t, jobs) - } - - for _, h := range firstHeartbeats { - generateUserJobs(h.UserID, time.Time(h.Time), jobs) + // Case 3: User doesn't have heartbeats at all + // -> Nothing to do } return nil } -func generateUserJobs(userId string, lastAggregation time.Time, jobs chan<- *AggregationJob) { - var from, to time.Time +func generateUserJobs(userId string, from time.Time, jobs chan<- *AggregationJob) { + var to time.Time + + // Go to next day of either user's first heartbeat or latest aggregation + from.Add(-1 * time.Second) + from = time.Date( + from.Year(), + from.Month(), + from.Day()+aggregateIntervalDays, + 0, 0, 0, 0, + from.Location(), + ) + + // Iteratively aggregate per-day summaries until end of yesterday is reached end := getStartOfToday().Add(-1 * time.Second) - - if lastAggregation.Hour() == 0 { - from = lastAggregation - } else { - from = time.Date( - lastAggregation.Year(), - lastAggregation.Month(), - lastAggregation.Day()+aggregateIntervalDays, - 0, 0, 0, 0, - lastAggregation.Location(), - ) - } - for from.Before(end) && to.Before(end) { to = time.Date( from.Year(), diff --git a/services/heartbeat.go b/services/heartbeat.go index 0123a73..e6e206a 100644 --- a/services/heartbeat.go +++ b/services/heartbeat.go @@ -38,8 +38,8 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User) return srv.augmented(heartbeats, user.ID) } -func (srv *HeartbeatService) GetFirstUserHeartbeats(userIds []string) ([]*models.Heartbeat, error) { - return srv.repository.GetFirstByUsers(userIds) +func (srv *HeartbeatService) GetFirstByUsers() ([]*models.TimeByUser, error) { + return srv.repository.GetFirstByUsers() } func (srv *HeartbeatService) DeleteBefore(t time.Time) error { diff --git a/services/summary.go b/services/summary.go index dd27ed0..e8b148e 100644 --- a/services/summary.go +++ b/services/summary.go @@ -4,14 +4,12 @@ import ( "crypto/md5" "errors" "github.com/muety/wakapi/config" + "github.com/muety/wakapi/models" "github.com/muety/wakapi/repositories" "github.com/patrickmn/go-cache" "math" "sort" - "strconv" "time" - - "github.com/muety/wakapi/models" ) const HeartbeatDiffThreshold = 2 * time.Minute @@ -24,6 +22,8 @@ type SummaryService struct { aliasService *AliasService } +type SummaryRetriever func(f, t time.Time, u *models.User) (*models.Summary, error) + func NewSummaryService(summaryRepo *repositories.SummaryRepository, heartbeatService *HeartbeatService, aliasService *AliasService) *SummaryService { return &SummaryService{ config: config.Get(), @@ -34,60 +34,98 @@ func NewSummaryService(summaryRepo *repositories.SummaryRepository, heartbeatSer } } -type Interval struct { - Start time.Time - End time.Time -} +// Public summary generation methods -// TODO: simplify! -func (srv *SummaryService) Construct(from, to time.Time, user *models.User, recompute bool) (*models.Summary, error) { - var existingSummaries []*models.Summary - var cacheKey string - - if recompute { - existingSummaries = make([]*models.Summary, 0) - } else { - cacheKey = getHash([]time.Time{from, to}, user) - if result, ok := srv.cache.Get(cacheKey); ok { - return result.(*models.Summary), nil - } - summaries, err := srv.GetByUserWithin(user, from, to) - if err != nil { - return nil, err - } - existingSummaries = summaries +func (srv *SummaryService) Aliased(from, to time.Time, user *models.User, f SummaryRetriever) (*models.Summary, error) { + // Check cache + cacheKey := srv.getHash(from.String(), to.String(), user.ID, "--aliased") + if cacheResult, ok := srv.cache.Get(cacheKey); ok { + return cacheResult.(*models.Summary), nil } - missingIntervals := getMissingIntervals(from, to, existingSummaries) + // Wrap alias resolution + resolve := func(t uint8, k string) string { + s, _ := srv.aliasService.GetAliasOrDefault(user.ID, t, k) + return s + } - heartbeats := make([]*models.Heartbeat, 0) + // Initialize alias resolver service + if err := srv.aliasService.LoadUserAliases(user.ID); err != nil { + return nil, err + } + + // Get actual summary + s, err := f(from, to, user) + if err != nil { + return nil, err + } + + // Post-process summary and cache it + summary := s.WithResolvedAliases(resolve) + srv.cache.SetDefault(cacheKey, summary) + return summary, nil +} + +func (srv *SummaryService) Retrieve(from, to time.Time, user *models.User) (*models.Summary, error) { + // Check cache + cacheKey := srv.getHash(from.String(), to.String(), user.ID, "--aliased") + if cacheResult, ok := srv.cache.Get(cacheKey); ok { + return cacheResult.(*models.Summary), nil + } + + // Get all already existing, pre-generated summaries that fall into the requested interval + summaries, err := srv.repository.GetByUserWithin(user, from, to) + if err != nil { + return nil, err + } + + // Generate missing slots (especially before and after existing summaries) from raw heartbeats + missingIntervals := srv.getMissingIntervals(from, to, summaries) for _, interval := range missingIntervals { - hb, err := srv.heartbeatService.GetAllWithin(interval.Start, interval.End, user) - if err != nil { + if s, err := srv.Summarize(interval.Start, interval.End, user); err == nil { + summaries = append(summaries, s) + } else { return nil, err } - heartbeats = append(heartbeats, hb...) + } + + // Merge existing and newly generated summary snippets + summary, err := srv.mergeSummaries(summaries) + if err != nil { + return nil, err + } + + // Cache 'em + srv.cache.SetDefault(cacheKey, summary) + return summary, nil +} + +func (srv *SummaryService) Summarize(from, to time.Time, user *models.User) (*models.Summary, error) { + // Initialize and fetch data + var heartbeats models.Heartbeats + if rawHeartbeats, err := srv.heartbeatService.GetAllWithin(from, to, user); err == nil { + heartbeats = rawHeartbeats + } else { + return nil, err } types := models.SummaryTypes() + typedAggregations := make(chan models.SummaryItemContainer) + defer close(typedAggregations) + for _, t := range types { + go srv.aggregateBy(heartbeats, t, typedAggregations) + } + + // Aggregate raw heartbeats by types in parallel and collect them var projectItems []*models.SummaryItem var languageItems []*models.SummaryItem var editorItems []*models.SummaryItem var osItems []*models.SummaryItem var machineItems []*models.SummaryItem - if err := srv.aliasService.LoadUserAliases(user.ID); err != nil { - return nil, err - } - - c := make(chan models.SummaryItemContainer) - for _, t := range types { - go srv.aggregateBy(heartbeats, t, user, c) - } - for i := 0; i < len(types); i++ { - item := <-c + item := <-typedAggregations switch item.Type { case models.SummaryProject: projectItems = item.Items @@ -101,31 +139,16 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco machineItems = item.Items } } - close(c) - realFrom, realTo := from, to - if len(existingSummaries) > 0 { - realFrom = existingSummaries[0].FromTime.T() - realTo = existingSummaries[len(existingSummaries)-1].ToTime.T() - - for _, summary := range existingSummaries { - summary.FillUnknown() - } - } - if len(heartbeats) > 0 { - t1, t2 := time.Time(heartbeats[0].Time), time.Time(heartbeats[len(heartbeats)-1].Time) - if t1.After(realFrom) && t1.Before(time.Date(realFrom.Year(), realFrom.Month(), realFrom.Day()+1, 0, 0, 0, 0, realFrom.Location())) { - realFrom = t1 - } - if t2.Before(realTo) && t2.After(time.Date(realTo.Year(), realTo.Month(), realTo.Day()-1, 0, 0, 0, 0, realTo.Location())) { - realTo = t2 - } + if heartbeats.Len() > 0 { + from = time.Time(heartbeats.First().Time) + to = time.Time(heartbeats.Last().Time) } - aggregatedSummary := &models.Summary{ + summary := &models.Summary{ UserID: user.ID, - FromTime: models.CustomTime(realFrom), - ToTime: models.CustomTime(realTo), + FromTime: models.CustomTime(from), + ToTime: models.CustomTime(to), Projects: projectItems, Languages: languageItems, Editors: editorItems, @@ -133,123 +156,32 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco Machines: machineItems, } - allSummaries := []*models.Summary{aggregatedSummary} - allSummaries = append(allSummaries, existingSummaries...) - - summary, err := mergeSummaries(allSummaries) - if err != nil { - return nil, err - } - - if cacheKey != "" { - srv.cache.SetDefault(cacheKey, summary) - } + summary.FillUnknown() return summary, nil } -func (srv *SummaryService) PostProcessWrapped(summary *models.Summary, err error) (*models.Summary, error) { - if err != nil { - return nil, err - } - return srv.PostProcess(summary), nil -} +// CRUD methods -func (srv *SummaryService) PostProcess(summary *models.Summary) *models.Summary { - updatedSummary := &models.Summary{ - ID: summary.ID, - UserID: summary.UserID, - FromTime: summary.FromTime, - ToTime: summary.ToTime, - } - - processAliases := func(origin []*models.SummaryItem) []*models.SummaryItem { - target := make([]*models.SummaryItem, 0) - - findItem := func(key string) *models.SummaryItem { - for _, item := range target { - if item.Key == key { - return item - } - } - return nil - } - - for _, item := range origin { - // Add all "top-level" items, i.e. such without aliases - if key, _ := srv.aliasService.GetAliasOrDefault(summary.UserID, item.Type, item.Key); key == item.Key { - target = append(target, item) - } - } - - for _, item := range origin { - // Add all remaining projects and merge with their alias - if key, _ := srv.aliasService.GetAliasOrDefault(summary.UserID, item.Type, item.Key); key != item.Key { - if targetItem := findItem(key); targetItem != nil { - targetItem.Total += item.Total - } else { - target = append(target, &models.SummaryItem{ - ID: item.ID, - SummaryID: item.SummaryID, - Type: item.Type, - Key: key, - Total: item.Total, - }) - } - } - } - - return target - } - - // Resolve aliases - updatedSummary.Projects = processAliases(summary.Projects) - updatedSummary.Editors = processAliases(summary.Editors) - updatedSummary.Languages = processAliases(summary.Languages) - updatedSummary.OperatingSystems = processAliases(summary.OperatingSystems) - updatedSummary.Machines = processAliases(summary.Machines) - - return updatedSummary -} - -func (srv *SummaryService) Insert(summary *models.Summary) error { - return srv.repository.Insert(summary) -} - -func (srv *SummaryService) GetByUserWithin(user *models.User, from, to time.Time) ([]*models.Summary, error) { - return srv.repository.GetByUserWithin(user, from, to) -} - -// Will return *models.Index objects with only user_id and to_time filled -func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) { - return srv.repository.GetLatestByUser() +func (srv *SummaryService) GetLatestByUser() ([]*models.TimeByUser, error) { + return srv.repository.GetLastByUser() } func (srv *SummaryService) DeleteByUser(userId string) error { return srv.repository.DeleteByUser(userId) } -func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, user *models.User, c chan models.SummaryItemContainer) { +func (srv *SummaryService) Insert(summary *models.Summary) error { + return srv.repository.Insert(summary) +} + +// Private summary generation and utility methods + +func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, c chan models.SummaryItemContainer) { durations := make(map[string]time.Duration) for i, h := range heartbeats { - var key string - switch summaryType { - case models.SummaryProject: - key = h.Project - case models.SummaryEditor: - key = h.Editor - case models.SummaryLanguage: - key = h.Language - case models.SummaryOS: - key = h.OperatingSystem - case models.SummaryMachine: - key = h.Machine - } - - if key == "" { - key = models.UnknownSummaryKey - } + key := h.GetKey(summaryType) if _, ok := durations[key]; !ok { durations[key] = time.Duration(0) @@ -287,43 +219,7 @@ func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryTy c <- models.SummaryItemContainer{Type: summaryType, Items: items} } -func getMissingIntervals(from, to time.Time, existingSummaries []*models.Summary) []*Interval { - if len(existingSummaries) == 0 { - return []*Interval{{from, to}} - } - - intervals := make([]*Interval, 0) - - // Pre - if from.Before(existingSummaries[0].FromTime.T()) { - intervals = append(intervals, &Interval{from, existingSummaries[0].FromTime.T()}) - } - - // Between - for i := 0; i < len(existingSummaries)-1; i++ { - t1, t2 := existingSummaries[i].ToTime.T(), existingSummaries[i+1].FromTime.T() - if t1.Equal(t2) { - continue - } - - // round to end of day / start of day, assuming that summaries are always generated on a per-day basis - td1 := time.Date(t1.Year(), t1.Month(), t1.Day()+1, 0, 0, 0, 0, t1.Location()) - td2 := time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, t2.Location()) - // one or more day missing in between? - if td1.Before(td2) { - intervals = append(intervals, &Interval{existingSummaries[i].ToTime.T(), existingSummaries[i+1].FromTime.T()}) - } - } - - // Post - if to.After(existingSummaries[len(existingSummaries)-1].ToTime.T()) { - intervals = append(intervals, &Interval{existingSummaries[len(existingSummaries)-1].ToTime.T(), to}) - } - - return intervals -} - -func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) { +func (srv *SummaryService) mergeSummaries(summaries []*models.Summary) (*models.Summary, error) { if len(summaries) < 1 { return nil, errors.New("no summaries given") } @@ -353,11 +249,11 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) { maxTime = s.ToTime.T() } - finalSummary.Projects = mergeSummaryItems(finalSummary.Projects, s.Projects) - finalSummary.Languages = mergeSummaryItems(finalSummary.Languages, s.Languages) - finalSummary.Editors = mergeSummaryItems(finalSummary.Editors, s.Editors) - finalSummary.OperatingSystems = mergeSummaryItems(finalSummary.OperatingSystems, s.OperatingSystems) - finalSummary.Machines = mergeSummaryItems(finalSummary.Machines, s.Machines) + finalSummary.Projects = srv.mergeSummaryItems(finalSummary.Projects, s.Projects) + finalSummary.Languages = srv.mergeSummaryItems(finalSummary.Languages, s.Languages) + finalSummary.Editors = srv.mergeSummaryItems(finalSummary.Editors, s.Editors) + finalSummary.OperatingSystems = srv.mergeSummaryItems(finalSummary.OperatingSystems, s.OperatingSystems) + finalSummary.Machines = srv.mergeSummaryItems(finalSummary.Machines, s.Machines) } finalSummary.FromTime = models.CustomTime(minTime) @@ -366,7 +262,7 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) { return finalSummary, nil } -func mergeSummaryItems(existing []*models.SummaryItem, new []*models.SummaryItem) []*models.SummaryItem { +func (srv *SummaryService) mergeSummaryItems(existing []*models.SummaryItem, new []*models.SummaryItem) []*models.SummaryItem { items := make(map[string]*models.SummaryItem) // Build map from existing @@ -396,11 +292,46 @@ func mergeSummaryItems(existing []*models.SummaryItem, new []*models.SummaryItem return itemList } -func getHash(times []time.Time, user *models.User) string { - digest := md5.New() - for _, t := range times { - digest.Write([]byte(strconv.Itoa(int(t.Unix())))) +func (srv *SummaryService) getMissingIntervals(from, to time.Time, summaries []*models.Summary) []*models.Interval { + if len(summaries) == 0 { + return []*models.Interval{{from, to}} + } + + intervals := make([]*models.Interval, 0) + + // Pre + if from.Before(summaries[0].FromTime.T()) { + intervals = append(intervals, &models.Interval{from, summaries[0].FromTime.T()}) + } + + // Between + for i := 0; i < len(summaries)-1; i++ { + t1, t2 := summaries[i].ToTime.T(), summaries[i+1].FromTime.T() + if t1.Equal(t2) { + continue + } + + // round to end of day / start of day, assuming that summaries are always generated on a per-day basis + td1 := time.Date(t1.Year(), t1.Month(), t1.Day()+1, 0, 0, 0, 0, t1.Location()) + td2 := time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, t2.Location()) + // one or more day missing in between? + if td1.Before(td2) { + intervals = append(intervals, &models.Interval{summaries[i].ToTime.T(), summaries[i+1].FromTime.T()}) + } + } + + // Post + if to.After(summaries[len(summaries)-1].ToTime.T()) { + intervals = append(intervals, &models.Interval{summaries[len(summaries)-1].ToTime.T(), to}) + } + + return intervals +} + +func (srv *SummaryService) getHash(args ...string) string { + digest := md5.New() + for _, a := range args { + digest.Write([]byte(a)) } - digest.Write([]byte(user.ID)) return string(digest.Sum(nil)) } diff --git a/version.txt b/version.txt index ace4423..15b989e 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.15.1 +1.16.0