1
0
mirror of https://github.com/muety/wakapi.git synced 2023-08-10 21:12:56 +03:00
wakapi/services/summary.go

472 lines
14 KiB
Go
Raw Normal View History

package services
import (
"errors"
"github.com/duke-git/lancet/v2/datetime"
"github.com/emvi/logbuch"
"github.com/leandro-lugaresi/hub"
"github.com/muety/wakapi/config"
"github.com/muety/wakapi/models"
"github.com/muety/wakapi/repositories"
2020-02-20 16:28:55 +03:00
"github.com/patrickmn/go-cache"
"sort"
"strings"
"time"
)
type SummaryService struct {
config *config.Config
cache *cache.Cache
eventBus *hub.Hub
repository repositories.ISummaryRepository
durationService IDurationService
aliasService IAliasService
projectLabelService IProjectLabelService
}
type SummaryRetriever func(f, t time.Time, u *models.User, filters *models.Filters) (*models.Summary, error)
func NewSummaryService(summaryRepo repositories.ISummaryRepository, durationService IDurationService, aliasService IAliasService, projectLabelService IProjectLabelService) *SummaryService {
srv := &SummaryService{
config: config.Get(),
cache: cache.New(24*time.Hour, 24*time.Hour),
eventBus: config.EventBus(),
repository: summaryRepo,
durationService: durationService,
aliasService: aliasService,
projectLabelService: projectLabelService,
}
sub1 := srv.eventBus.Subscribe(0, config.TopicProjectLabel)
go func(sub *hub.Subscription) {
for m := range sub.Receiver {
srv.invalidateUserCache(m.Fields[config.FieldUserId].(string))
}
}(&sub1)
return srv
}
// Public summary generation methods
// Aliased retrieves or computes a new summary based on the given SummaryRetriever and augments it with entity aliases and project labels
func (srv *SummaryService) Aliased(from, to time.Time, user *models.User, f SummaryRetriever, filters *models.Filters, skipCache bool) (*models.Summary, error) {
// Check cache
cacheKey := srv.getHash(from.String(), to.String(), user.ID, filters.Hash(), "--aliased")
if cacheResult, ok := srv.cache.Get(cacheKey); ok && !skipCache {
return cacheResult.(*models.Summary), nil
}
2020-02-20 16:28:55 +03:00
// Resolver functions
resolveAliases := srv.getAliasResolver(user)
resolveAliasesReverse := srv.getAliasReverseResolver(user)
resolveProjectLabelsReverse := srv.getProjectLabelsReverseResolver(user)
// Post-process filters
if filters != nil {
filters = filters.WithAliases(resolveAliasesReverse)
filters = filters.WithProjectLabels(resolveProjectLabelsReverse)
}
// Initialize alias resolver service
if err := srv.aliasService.InitializeUser(user.ID); err != nil {
return nil, err
}
// Get actual summary
s, err := f(from, to, user, filters)
if err != nil {
return nil, err
}
// Post-process summary and cache it
summary := s.WithResolvedAliases(resolveAliases)
summary = srv.withProjectLabels(summary)
summary.FillBy(models.SummaryProject, models.SummaryLabel) // first fill up labels from projects
summary.FillMissing() // then, full up types which are entirely missing
if withBranches := filters != nil && filters.Project != nil && filters.Project.Exists(); !withBranches {
summary.Branches = nil
}
srv.cache.SetDefault(cacheKey, summary)
return summary.Sorted(), nil
}
func (srv *SummaryService) Retrieve(from, to time.Time, user *models.User, filters *models.Filters) (*models.Summary, error) {
summaries := make([]*models.Summary, 0)
// Filtered summaries are not persisted currently
if filters == nil || filters.IsEmpty() {
// Get all already existing, pre-generated summaries that fall into the requested interval
result, err := srv.repository.GetByUserWithin(user, from, to)
if err == nil {
summaries = result
} else {
return nil, err
}
}
// Generate missing slots (especially before and after existing summaries) from durations (formerly raw heartbeats)
missingIntervals := srv.getMissingIntervals(from, to, summaries, false)
for _, interval := range missingIntervals {
if s, err := srv.Summarize(interval.Start, interval.End, user, filters); err == nil {
if len(missingIntervals) > 2 && s.FromTime.T().Equal(s.ToTime.T()) {
// little hack here: GetAllWithin will query for >= from_date
// however, for "in-between" / intra-day missing intervals, we want strictly > from_date to prevent double-counting
// to not have to rewrite many interfaces, we skip these summaries here
continue
}
summaries = append(summaries, s)
} else {
return nil, err
}
}
// Merge existing and newly generated summary snippets
summary, err := srv.mergeSummaries(summaries)
if err != nil {
return nil, err
}
2019-05-19 22:00:19 +03:00
return summary.Sorted(), nil
}
2019-05-19 22:00:19 +03:00
func (srv *SummaryService) Summarize(from, to time.Time, user *models.User, filters *models.Filters) (*models.Summary, error) {
// Initialize and fetch data
durations, err := srv.durationService.Get(from, to, user, filters)
if err != nil {
return nil, err
}
types := models.PersistedSummaryTypes()
if filters != nil && filters.Project != nil && filters.Project.Exists() {
types = append(types, models.SummaryBranch)
}
typedAggregations := make(chan models.SummaryItemContainer)
defer close(typedAggregations)
2019-05-19 22:00:19 +03:00
for _, t := range types {
go srv.aggregateBy(durations, t, typedAggregations)
2019-05-19 22:00:19 +03:00
}
// Aggregate durations (formerly raw heartbeats) by types in parallel and collect them
var projectItems []*models.SummaryItem
var languageItems []*models.SummaryItem
var editorItems []*models.SummaryItem
var osItems []*models.SummaryItem
var machineItems []*models.SummaryItem
var branchItems []*models.SummaryItem
2019-05-19 22:00:19 +03:00
for i := 0; i < len(types); i++ {
item := <-typedAggregations
2019-05-19 22:00:19 +03:00
switch item.Type {
case models.SummaryProject:
projectItems = item.Items
case models.SummaryLanguage:
languageItems = item.Items
case models.SummaryEditor:
editorItems = item.Items
case models.SummaryOS:
osItems = item.Items
case models.SummaryMachine:
machineItems = item.Items
case models.SummaryBranch:
branchItems = item.Items
2019-05-19 22:00:19 +03:00
}
}
if durations.Len() > 0 {
from = time.Time(durations.First().Time)
to = time.Time(durations.Last().Time)
}
summary := &models.Summary{
UserID: user.ID,
FromTime: models.CustomTime(from),
ToTime: models.CustomTime(to),
2019-05-19 22:00:19 +03:00
Projects: projectItems,
Languages: languageItems,
Editors: editorItems,
OperatingSystems: osItems,
Machines: machineItems,
Branches: branchItems,
NumHeartbeats: durations.TotalNumHeartbeats(),
}
2020-02-20 16:28:55 +03:00
return summary.Sorted(), nil
}
// CRUD methods
func (srv *SummaryService) GetLatestByUser() ([]*models.TimeByUser, error) {
return srv.repository.GetLastByUser()
}
func (srv *SummaryService) DeleteByUser(userId string) error {
srv.invalidateUserCache(userId)
return srv.repository.DeleteByUser(userId)
}
func (srv *SummaryService) Insert(summary *models.Summary) error {
srv.invalidateUserCache(summary.UserID)
return srv.repository.Insert(summary)
2019-10-10 17:47:19 +03:00
}
// Private summary generation and utility methods
func (srv *SummaryService) aggregateBy(durations []*models.Duration, summaryType uint8, c chan models.SummaryItemContainer) {
mapping := make(map[string]time.Duration)
for _, d := range durations {
mapping[d.GetKey(summaryType)] += d.Duration
}
items := make([]*models.SummaryItem, 0)
for k, v := range mapping {
items = append(items, &models.SummaryItem{
Key: k,
Total: v / time.Second,
Type: summaryType,
})
}
sort.Slice(items, func(i, j int) bool {
return items[i].Total > items[j].Total
})
2019-05-19 22:00:19 +03:00
c <- models.SummaryItemContainer{Type: summaryType, Items: items}
}
func (srv *SummaryService) withProjectLabels(summary *models.Summary) *models.Summary {
newEntry := func(key string, total time.Duration) *models.SummaryItem {
return &models.SummaryItem{
Type: models.SummaryLabel,
Key: key,
Total: total,
}
}
allLabels, err := srv.projectLabelService.GetByUser(summary.UserID)
if err != nil {
logbuch.Error("failed to retrieve project labels for user summary ('%s', '%s', '%s')", summary.UserID, summary.FromTime.String(), summary.ToTime.String())
return summary
}
mappedProjects := make(map[string]*models.SummaryItem, len(summary.Projects))
for _, p := range summary.Projects {
mappedProjects[p.Key] = p
}
var totalLabelTime time.Duration
labelMap := make(map[string]*models.SummaryItem, 0)
for _, l := range allLabels {
if p, ok := mappedProjects[l.ProjectKey]; ok {
if _, ok2 := labelMap[l.Label]; !ok2 {
labelMap[l.Label] = newEntry(l.Label, 0)
}
labelMap[l.Label].Total += p.Total
totalLabelTime += p.Total
}
}
//labelMap[models.DefaultProjectLabel] = newEntry(models.DefaultProjectLabel, summary.TotalTimeBy(models.SummaryProject) / time.Second-totalLabelTime)
labels := make([]*models.SummaryItem, 0, len(labelMap))
for _, v := range labelMap {
2021-06-12 13:06:24 +03:00
if v.Total > 0 {
labels = append(labels, v)
}
}
summary.Labels = labels
return summary
}
func (srv *SummaryService) mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
if len(summaries) < 1 {
return nil, errors.New("no summaries given")
}
var minTime, maxTime time.Time
minTime = time.Now()
finalSummary := &models.Summary{
UserID: summaries[0].UserID,
Projects: make([]*models.SummaryItem, 0),
Languages: make([]*models.SummaryItem, 0),
Editors: make([]*models.SummaryItem, 0),
OperatingSystems: make([]*models.SummaryItem, 0),
Machines: make([]*models.SummaryItem, 0),
Labels: make([]*models.SummaryItem, 0),
Branches: make([]*models.SummaryItem, 0),
}
var processed = map[time.Time]bool{}
for _, s := range summaries {
hash := s.FromTime.T()
if _, found := processed[hash]; found {
logbuch.Warn("summary from %v to %v (user '%s') was attempted to be processed more often than once", s.FromTime, s.ToTime, s.UserID)
continue
}
if s.UserID != finalSummary.UserID {
return nil, errors.New("users don't match")
}
if s.FromTime.T().Before(minTime) {
minTime = s.FromTime.T()
}
if s.ToTime.T().After(maxTime) {
maxTime = s.ToTime.T()
}
finalSummary.Projects = srv.mergeSummaryItems(finalSummary.Projects, s.Projects)
finalSummary.Languages = srv.mergeSummaryItems(finalSummary.Languages, s.Languages)
finalSummary.Editors = srv.mergeSummaryItems(finalSummary.Editors, s.Editors)
finalSummary.OperatingSystems = srv.mergeSummaryItems(finalSummary.OperatingSystems, s.OperatingSystems)
finalSummary.Machines = srv.mergeSummaryItems(finalSummary.Machines, s.Machines)
finalSummary.Labels = srv.mergeSummaryItems(finalSummary.Labels, s.Labels)
finalSummary.Branches = srv.mergeSummaryItems(finalSummary.Branches, s.Branches)
finalSummary.NumHeartbeats += s.NumHeartbeats
processed[hash] = true
}
finalSummary.FromTime = models.CustomTime(minTime)
finalSummary.ToTime = models.CustomTime(maxTime)
return finalSummary, nil
}
func (srv *SummaryService) mergeSummaryItems(existing []*models.SummaryItem, new []*models.SummaryItem) []*models.SummaryItem {
items := make(map[string]*models.SummaryItem)
// Build map from existing
for _, item := range existing {
items[item.Key] = item
}
for _, item := range new {
if it, ok := items[item.Key]; !ok {
items[item.Key] = item
} else {
(*it).Total += item.Total
}
}
var i int
itemList := make([]*models.SummaryItem, len(items))
for k, v := range items {
itemList[i] = &models.SummaryItem{Key: k, Total: v.Total, Type: v.Type}
i++
}
sort.Slice(itemList, func(i, j int) bool {
return itemList[i].Total > itemList[j].Total
})
return itemList
}
2020-02-20 16:28:55 +03:00
func (srv *SummaryService) getMissingIntervals(from, to time.Time, summaries []*models.Summary, precise bool) []*models.Interval {
if len(summaries) == 0 {
return []*models.Interval{{from, to}}
}
intervals := make([]*models.Interval, 0)
// Pre
if from.Before(summaries[0].FromTime.T()) {
intervals = append(intervals, &models.Interval{Start: from, End: summaries[0].FromTime.T()})
}
// Between
for i := 0; i < len(summaries)-1; i++ {
t1, t2 := summaries[i].ToTime.T(), summaries[i+1].FromTime.T()
if t1.Equal(t2) || t1.Equal(to) || t1.After(to) {
continue
}
td1 := t1
td2 := t2
// round to end of day / start of day, assuming that summaries are always generated on a per-day basis
// we assume that, if summary for any time range within a day is present, no further heartbeats exist on that day before 'from' and after 'to' time of that summary
// this requires that a summary exists for every single day in a year and none is skipped, which shouldn't ever happen
// non-precise mode is mainly for speed when fetching summaries over large intervals and trades speed for summary accuracy / comprehensiveness
if !precise {
td1 = datetime.BeginOfDay(t1).AddDate(0, 0, 1)
td2 = datetime.BeginOfDay(t2)
// we always want to jump to beginning of next day
// however, if left summary ends already at midnight, we would instead jump to beginning of second-next day -> go back again
if td1.AddDate(0, 0, 1).Equal(t1) {
td1 = td1.Add(-1 * time.Hour)
}
}
// one or more day missing in between?
if td1.Before(td2) {
intervals = append(intervals, &models.Interval{Start: t1, End: t2})
}
}
// Post
if to.After(summaries[len(summaries)-1].ToTime.T()) {
intervals = append(intervals, &models.Interval{Start: summaries[len(summaries)-1].ToTime.T(), End: to})
}
return intervals
}
func (srv *SummaryService) getHash(args ...string) string {
return strings.Join(args, "__")
}
func (srv *SummaryService) invalidateUserCache(userId string) {
for key := range srv.cache.Items() {
if strings.Contains(key, userId) {
srv.cache.Delete(key)
}
2020-02-20 16:28:55 +03:00
}
}
func (srv *SummaryService) getAliasResolver(user *models.User) models.AliasResolver {
return func(t uint8, k string) string {
s, _ := srv.aliasService.GetAliasOrDefault(user.ID, t, k)
return s
}
}
func (srv *SummaryService) getAliasReverseResolver(user *models.User) models.AliasReverseResolver {
return func(t uint8, k string) []string {
aliases, err := srv.aliasService.GetByUserAndKeyAndType(user.ID, k, t)
if err != nil {
aliases = []*models.Alias{}
}
aliasStrings := make([]string, 0, len(aliases))
for _, a := range aliases {
aliasStrings = append(aliasStrings, a.Value)
}
return aliasStrings
}
}
func (srv *SummaryService) getProjectLabelsReverseResolver(user *models.User) models.ProjectLabelReverseResolver {
return func(k string) []string {
var labels []*models.ProjectLabel
allLabels, err := srv.projectLabelService.GetByUserGroupedInverted(user.ID)
if err == nil {
if l, ok := allLabels[k]; ok {
labels = l
}
}
projectStrings := make([]string, 0, len(labels))
for _, l := range labels {
projectStrings = append(projectStrings, l.ProjectKey)
}
return projectStrings
}
}