mirror
/
wakapi
mirror of https://github.com/muety/wakapi.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix summary merging. 

Rename some methods.
Use pointers for structs and none for primitives.
This commit is contained in:
Ferdinand Mütsch 2019-10-11 08:00:02 +02:00
parent 3696622493
commit 37b02ff32c
7 changed files with 123 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
main.go
View File

@ -103,10 +103,10 @@ func main() {
heartbeatSrvc := &services.HeartbeatService{config, db}
userSrvc := &services.UserService{config, db}
summarySrvc := &services.SummaryService{config, db, heartbeatSrvc, aliasSrvc}
aggregationSrvc := &services.AggregationService{config, db, userSrvc, summarySrvc, heartbeatSrvc}
_ = &services.AggregationService{config, db, userSrvc, summarySrvc, heartbeatSrvc}
// DEBUG ONLY !!!
aggregationSrvc.Start(time.Second)
//aggregationSrvc.Start(time.Second)
// Handlers
heartbeatHandler := &routes.HeartbeatHandler{HeartbeatSrvc: heartbeatSrvc}

28
models/heartbeat.go
View File

@ -13,24 +13,24 @@ import (
type HeartbeatReqTime time.Time
type Heartbeat struct {
ID uint `gorm:"primary_key"`
User *User `json:"-" gorm:"not null"`
UserID string `json:"-" gorm:"not null; index:idx_time_user"`
Entity string `json:"entity" gorm:"not null; index:idx_entity"`
Type string `json:"type"`
Category string `json:"category"`
Project string `json:"project"`
Branch string `json:"branch"`
Language string `json:"language" gorm:"index:idx_language"`
IsWrite bool `json:"is_write"`
Editor string `json:"editor"`
OperatingSystem string `json:"operating_system"`
Time *HeartbeatReqTime `json:"time" gorm:"type:timestamp; default:now(); index:idx_time,idx_time_user"`
ID uint `gorm:"primary_key"`
User *User `json:"-" gorm:"not null"`
UserID string `json:"-" gorm:"not null; index:idx_time_user"`
Entity string `json:"entity" gorm:"not null; index:idx_entity"`
Type string `json:"type"`
Category string `json:"category"`
Project string `json:"project"`
Branch string `json:"branch"`
Language string `json:"language" gorm:"index:idx_language"`
IsWrite bool `json:"is_write"`
Editor string `json:"editor"`
OperatingSystem string `json:"operating_system"`
Time HeartbeatReqTime `json:"time" gorm:"type:timestamp; default:now(); index:idx_time,idx_time_user"`
languageRegex *regexp.Regexp
}
func (h *Heartbeat) Valid() bool {
return h.User != nil && h.UserID != "" && h.Time != nil
return h.User != nil && h.UserID != "" && h.Time != HeartbeatReqTime(time.Time{})
}
func (h *Heartbeat) Augment(customLangs map[string]string) {

18
models/summary.go
View File

@ -13,14 +13,14 @@ const (
)
type Summary struct {
ID uint `json:"-" gorm:"primary_key"`
UserID string `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
FromTime *time.Time `json:"from" gorm:"not null; type:timestamp; default:now(); index:idx_time_summary_user"`
ToTime *time.Time `json:"to" gorm:"not null; type:timestamp; default:now(); index:idx_time_summary_user"`
Projects []SummaryItem `json:"projects"`
Languages []SummaryItem `json:"languages"`
Editors []SummaryItem `json:"editors"`
OperatingSystems []SummaryItem `json:"operating_systems"`
ID uint `json:"-" gorm:"primary_key"`
UserID string `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
FromTime time.Time `json:"from" gorm:"not null; type:timestamp; default:now(); index:idx_time_summary_user"`
ToTime time.Time `json:"to" gorm:"not null; type:timestamp; default:now(); index:idx_time_summary_user"`
Projects []*SummaryItem `json:"projects"`
Languages []*SummaryItem `json:"languages"`
Editors []*SummaryItem `json:"editors"`
OperatingSystems []*SummaryItem `json:"operating_systems"`
}
type SummaryItem struct {
@ -33,5 +33,5 @@ type SummaryItem struct {
type SummaryItemContainer struct {
Type uint8
Items []SummaryItem
Items []*SummaryItem
}

2
routes/summary.go
View File

@ -76,7 +76,7 @@ func (h *SummaryHandler) Get(w http.ResponseWriter, r *http.Request) {
cacheKey := getHash([]time.Time{from, to}, user)
if cachedSummary, ok := h.Cache.Get(cacheKey); !ok {
// Cache Miss
summary, err = h.SummarySrvc.CreateSummary(from, to, user) // 'to' is always constant
summary, err = h.SummarySrvc.Construct(from, to, user) // 'to' is always constant
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
return

12
services/aggregation.go
View File

@ -58,7 +58,7 @@ func (srv *AggregationService) Start(interval time.Duration) {
func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) {
for job := range jobs {
if summary, err := srv.SummaryService.CreateSummary(job.From, job.To, &models.User{ID: job.UserID}); err != nil {
if summary, err := srv.SummaryService.Construct(job.From, job.To, &models.User{ID: job.UserID}); err != nil {
log.Printf("Failed to generate summary (%v, %v, %s) – %v.", job.From, job.To, job.UserID, err)
} else {
summaries <- summary
@ -68,7 +68,7 @@ func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summar
func (srv *AggregationService) persistWorker(summaries <-chan *models.Summary) {
for summary := range summaries {
if err := srv.SummaryService.SaveSummary(summary); err != nil {
if err := srv.SummaryService.Insert(summary); err != nil {
log.Printf("Failed to save summary (%v, %v, %s) – %v.", summary.UserID, summary.FromTime, summary.ToTime, err)
}
}
@ -80,12 +80,12 @@ func (srv *AggregationService) generateJobs(jobs chan<- *AggregationJob) error {
return err
}
latestSummaries, err := srv.SummaryService.GetLatestUserSummaries()
latestSummaries, err := srv.SummaryService.GetLatestByUser()
if err != nil {
return err
}
userSummaryTimes := make(map[string]*time.Time)
userSummaryTimes := make(map[string]time.Time)
for _, s := range latestSummaries {
userSummaryTimes[s.UserID] = s.ToTime
}
@ -103,11 +103,11 @@ func (srv *AggregationService) generateJobs(jobs chan<- *AggregationJob) error {
}
for id, t := range userSummaryTimes {
generateUserJobs(id, *t, jobs)
generateUserJobs(id, t, jobs)
}
for _, h := range firstHeartbeats {
generateUserJobs(h.UserID, time.Time(*(h.Time)), jobs)
generateUserJobs(h.UserID, time.Time(h.Time), jobs)
}
return nil

1
services/heartbeat.go
View File

@ -40,6 +40,7 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
return heartbeats, nil
}
// Will return *models.Heartbeat object with only user_id and time fields filled
func (srv *HeartbeatService) GetFirstUserHeartbeats(userIds []string) ([]*models.Heartbeat, error) {
var heartbeats []*models.Heartbeat
if err := srv.Db.

180
services/summary.go
View File

@ -23,8 +23,7 @@ type Interval struct {
End time.Time
}
// TODO: Rename methods to clarify difference between generating a new summary from old summaries and heartbeats, like this method, and only retrieving a persisted summary from database, like GetByUserWithin
func (srv *SummaryService) CreateSummary(from, to time.Time, user *models.User) (*models.Summary, error) {
func (srv *SummaryService) Construct(from, to time.Time, user *models.User) (*models.Summary, error) {
existingSummaries, err := srv.GetByUserWithin(user, from, to)
if err != nil {
return nil, err
@ -43,10 +42,10 @@ func (srv *SummaryService) CreateSummary(from, to time.Time, user *models.User)
types := []uint8{models.SummaryProject, models.SummaryLanguage, models.SummaryEditor, models.SummaryOS}
var projectItems []models.SummaryItem
var languageItems []models.SummaryItem
var editorItems []models.SummaryItem
var osItems []models.SummaryItem
var projectItems []*models.SummaryItem
var languageItems []*models.SummaryItem
var editorItems []*models.SummaryItem
var osItems []*models.SummaryItem
if err := srv.AliasService.LoadUserAliases(user.ID); err != nil {
return nil, err
@ -74,8 +73,8 @@ func (srv *SummaryService) CreateSummary(from, to time.Time, user *models.User)
aggregatedSummary := &models.Summary{
UserID: user.ID,
FromTime: &from,
ToTime: &to,
FromTime: from,
ToTime: to,
Projects: projectItems,
Languages: languageItems,
Editors: editorItems,
@ -93,79 +92,7 @@ func (srv *SummaryService) CreateSummary(from, to time.Time, user *models.User)
return summary, nil
}
func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
if len(summaries) < 1 {
return nil, errors.New("no summaries given")
}
var minTime, maxTime time.Time
minTime = time.Now()
finalSummary := &models.Summary{
UserID: summaries[0].UserID,
Projects: make([]models.SummaryItem, 0),
Languages: make([]models.SummaryItem, 0),
Editors: make([]models.SummaryItem, 0),
OperatingSystems: make([]models.SummaryItem, 0),
}
for _, s := range summaries {
if s.UserID != finalSummary.UserID {
return nil, errors.New("users don't match")
}
if s.FromTime.Before(minTime) {
minTime = *(s.FromTime)
}
if s.ToTime.After(maxTime) {
maxTime = *(s.ToTime)
}
// TODO: Multi-thread ?
finalSummary.Projects = mergeSummaryItems(&(finalSummary.Projects), &(s.Projects))
finalSummary.Languages = mergeSummaryItems(&(finalSummary.Languages), &(s.Languages))
finalSummary.Editors = mergeSummaryItems(&(finalSummary.Editors), &(s.Editors))
finalSummary.OperatingSystems = mergeSummaryItems(&(finalSummary.OperatingSystems), &(s.OperatingSystems))
}
finalSummary.FromTime = &minTime
finalSummary.ToTime = &maxTime
return finalSummary, nil
}
func mergeSummaryItems(existing *[]models.SummaryItem, new *[]models.SummaryItem) []models.SummaryItem {
items := make(map[string]*models.SummaryItem)
// Build map from existing
for _, item := range *existing {
items[item.Key] = &item
}
for _, item := range *new {
if it, ok := items[item.Key]; !ok {
items[item.Key] = &item
} else {
(*it).Total += item.Total
}
}
var i int
itemList := make([]models.SummaryItem, len(items))
for k, v := range items {
itemList[i] = models.SummaryItem{Key: k, Total: v.Total, Type: v.Type}
i++
}
sort.Slice(itemList, func(i, j int) bool {
return itemList[i].Total > itemList[j].Total
})
return itemList
}
func (srv *SummaryService) SaveSummary(summary *models.Summary) error {
func (srv *SummaryService) Insert(summary *models.Summary) error {
fmt.Println("Saving summary", summary)
if err := srv.Db.Create(summary).Error; err != nil {
return err
@ -189,7 +116,8 @@ func (srv *SummaryService) GetByUserWithin(user *models.User, from, to time.Time
return summaries, nil
}
func (srv *SummaryService) GetLatestUserSummaries() ([]*models.Summary, error) {
// Will return *models.Summary objects with only user_id and to_time filled
func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) {
var summaries []*models.Summary
if err := srv.Db.
Table("summaries").
@ -238,9 +166,9 @@ func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryTy
durations[key] += time.Duration(int64(timeThresholded))
}
items := make([]models.SummaryItem, 0)
items := make([]*models.SummaryItem, 0)
for k, v := range durations {
items = append(items, models.SummaryItem{
items = append(items, &models.SummaryItem{
Key: k,
Total: v / time.Second,
Type: summaryType,
@ -262,21 +190,93 @@ func getMissingIntervals(from, to time.Time, existingSummaries []*models.Summary
intervals := make([]*Interval, 0)
// Pre
if from.Before(*(existingSummaries[0].FromTime)) {
intervals = append(intervals, &Interval{from, *(existingSummaries[0].FromTime)})
if from.Before(existingSummaries[0].FromTime) {
intervals = append(intervals, &Interval{from, existingSummaries[0].FromTime})
}
// Between
for i := 0; i < len(existingSummaries)-1; i++ {
if existingSummaries[i].ToTime.Before(*(existingSummaries[i+1].FromTime)) {
intervals = append(intervals, &Interval{*(existingSummaries[i].ToTime), *(existingSummaries[i+1].FromTime)})
if existingSummaries[i].ToTime.Before(existingSummaries[i+1].FromTime) {
intervals = append(intervals, &Interval{existingSummaries[i].ToTime, existingSummaries[i+1].FromTime})
}
}
// Post
if to.After(*(existingSummaries[len(existingSummaries)-1].ToTime)) {
intervals = append(intervals, &Interval{to, *(existingSummaries[len(existingSummaries)-1].ToTime)})
if to.After(existingSummaries[len(existingSummaries)-1].ToTime) {
intervals = append(intervals, &Interval{to, existingSummaries[len(existingSummaries)-1].ToTime})
}
return intervals
}
func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
if len(summaries) < 1 {
return nil, errors.New("no summaries given")
}
var minTime, maxTime time.Time
minTime = time.Now()
finalSummary := &models.Summary{
UserID: summaries[0].UserID,
Projects: make([]*models.SummaryItem, 0),
Languages: make([]*models.SummaryItem, 0),
Editors: make([]*models.SummaryItem, 0),
OperatingSystems: make([]*models.SummaryItem, 0),
}
for _, s := range summaries {
if s.UserID != finalSummary.UserID {
return nil, errors.New("users don't match")
}
if s.FromTime.Before(minTime) {
minTime = s.FromTime
}
if s.ToTime.After(maxTime) {
maxTime = s.ToTime
}
// TODO: Multi-thread ?
finalSummary.Projects = mergeSummaryItems(finalSummary.Projects, s.Projects)
finalSummary.Languages = mergeSummaryItems(finalSummary.Languages, s.Languages)
finalSummary.Editors = mergeSummaryItems(finalSummary.Editors, s.Editors)
finalSummary.OperatingSystems = mergeSummaryItems(finalSummary.OperatingSystems, s.OperatingSystems)
}
finalSummary.FromTime = minTime
finalSummary.ToTime = maxTime
return finalSummary, nil
}
func mergeSummaryItems(existing []*models.SummaryItem, new []*models.SummaryItem) []*models.SummaryItem {
items := make(map[string]*models.SummaryItem)
// Build map from existing
for _, item := range existing {
items[item.Key] = item
}
for _, item := range new {
if it, ok := items[item.Key]; !ok {
items[item.Key] = item
} else {
(*it).Total += item.Total
}
}
var i int
itemList := make([]*models.SummaryItem, len(items))
for k, v := range items {
itemList[i] = &models.SummaryItem{Key: k, Total: v.Total, Type: v.Type}
i++
}
sort.Slice(itemList, func(i, j int) bool {
return itemList[i].Total > itemList[j].Total
})
return itemList
}