From be906805e7627bc74d5882c747db8bc25dd8195a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Sun, 19 May 2019 19:49:27 +0200 Subject: [PATCH] Major refactorings. Introduce summaries. --- config.ini | 5 +- main.go | 30 ++++---- middlewares/authenticate.go | 6 +- models/aggregation.go | 45 ------------ models/config.go | 15 ++-- models/heartbeat.go | 6 +- models/summary.go | 28 +++++++ routes/aggregation.go | 49 ------------- routes/heartbeat.go | 12 +-- routes/summary.go | 41 +++++++++++ services/aggregation.go | 142 ------------------------------------ services/heartbeat.go | 9 ++- services/summary.go | 74 +++++++++++++++++++ utils/http.go | 14 ++++ 14 files changed, 194 insertions(+), 282 deletions(-) delete mode 100644 models/aggregation.go create mode 100644 models/summary.go delete mode 100644 routes/aggregation.go create mode 100644 routes/summary.go delete mode 100644 services/aggregation.go create mode 100644 services/summary.go create mode 100644 utils/http.go diff --git a/config.ini b/config.ini index 056e07a..0af95eb 100644 --- a/config.ini +++ b/config.ini @@ -1,5 +1,2 @@ [server] -port = 3000 - -[data] -aggregation_interval = 24h \ No newline at end of file +port = 3000 \ No newline at end of file diff --git a/main.go b/main.go index e009031..e72c031 100644 --- a/main.go +++ b/main.go @@ -42,18 +42,18 @@ func readConfig() *models.Config { log.Fatal(fmt.Sprintf("Fail to read file: %v", err)) } - port := cfg.Section("server").Key("port").MustInt() - intervalStr := cfg.Section("data").Key("aggregation_interval").String() - interval, _ := time.ParseDuration(intervalStr) + port, err := strconv.Atoi(os.Getenv("PORT")) + if err != nil { + port = cfg.Section("server").Key("port").MustInt() + } return &models.Config{ - Port: port, - DbHost: dbHost, - DbUser: dbUser, - DbPassword: dbPassword, - DbName: dbName, - DbDialect: "mysql", - AggregationInterval: interval, + Port: port, + DbHost: dbHost, + DbUser: dbUser, + DbPassword: dbPassword, + DbName: dbName, + DbDialect: "mysql", } } @@ -73,17 +73,15 @@ func main() { // Migrate database schema db.AutoMigrate(&models.User{}) db.AutoMigrate(&models.Heartbeat{}).AddForeignKey("user_id", "users(id)", "RESTRICT", "RESTRICT") - db.AutoMigrate(&models.Aggregation{}).AddForeignKey("user_id", "users(id)", "RESTRICT", "RESTRICT") - db.AutoMigrate(&models.AggregationItem{}).AddForeignKey("aggregation_id", "aggregations(id)", "RESTRICT", "RESTRICT") // Services heartbeatSrvc := &services.HeartbeatService{config, db} userSrvc := &services.UserService{config, db} - aggregationSrvc := &services.AggregationService{config, db, heartbeatSrvc} + summarySrvc := &services.SummaryService{config, db, heartbeatSrvc} // Handlers heartbeatHandler := &routes.HeartbeatHandler{HeartbeatSrvc: heartbeatSrvc} - aggregationHandler := &routes.AggregationHandler{AggregationSrvc: aggregationSrvc} + summaryHandler := &routes.SummaryHandler{SummarySrvc: summarySrvc} // Middlewares authenticate := &middlewares.AuthenticateMiddleware{UserSrvc: userSrvc} @@ -96,8 +94,8 @@ func main() { heartbeats := apiRouter.Path("/heartbeat").Subrouter() heartbeats.Methods("POST").HandlerFunc(heartbeatHandler.Post) - aggreagations := apiRouter.Path("/aggregation").Subrouter() - aggreagations.Methods("GET").HandlerFunc(aggregationHandler.Get) + aggreagations := apiRouter.Path("/summary").Subrouter() + aggreagations.Methods("GET").HandlerFunc(summaryHandler.Get) // Sub-Routes Setup router.PathPrefix("/api").Handler(negroni.Classic().With( diff --git a/middlewares/authenticate.go b/middlewares/authenticate.go index 26f65ba..305de56 100644 --- a/middlewares/authenticate.go +++ b/middlewares/authenticate.go @@ -17,19 +17,19 @@ type AuthenticateMiddleware struct { func (m *AuthenticateMiddleware) Handle(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { authHeader := strings.Split(r.Header.Get("Authorization"), " ") if len(authHeader) != 2 { - w.WriteHeader(401) + w.WriteHeader(http.StatusUnauthorized) return } key, err := base64.StdEncoding.DecodeString(authHeader[1]) if err != nil { - w.WriteHeader(401) + w.WriteHeader(http.StatusUnauthorized) return } user, err := m.UserSrvc.GetUserByKey(strings.TrimSpace(string(key))) if err != nil { - w.WriteHeader(401) + w.WriteHeader(http.StatusUnauthorized) return } diff --git a/models/aggregation.go b/models/aggregation.go deleted file mode 100644 index fffea6c..0000000 --- a/models/aggregation.go +++ /dev/null @@ -1,45 +0,0 @@ -package models - -import ( - "database/sql/driver" - "time" - - "github.com/jinzhu/gorm" -) - -const ( - NAggregationTypes uint8 = 4 - AggregationProject uint8 = 0 - AggregationLanguage uint8 = 1 - AggregationEditor uint8 = 2 - AggregationOS uint8 = 3 -) - -type Aggregation struct { - gorm.Model - User *User `gorm:"not null; association_foreignkey:ID"` - UserID string `gorm:"not null; index:idx_user,idx_type_time_user"` - FromTime *time.Time `gorm:"not null; index:idx_from,idx_type_time_user; default:now()"` - ToTime *time.Time `gorm:"not null; index:idx_to,idx_type_time_user; default:now()"` - Duration time.Duration `gorm:"-"` - Type uint8 `gorm:"not null; index:idx_type,idx_type_time_user"` - Items []AggregationItem -} - -type AggregationItem struct { - ID uint `gorm:"primary_key; auto_increment"` - AggregationID uint `gorm:"not null; association_foreignkey:ID"` - Key string `gorm:"not null"` - Total ScannableDuration -} - -type ScannableDuration time.Duration - -func (d *ScannableDuration) Scan(value interface{}) error { - *d = ScannableDuration(*d) * ScannableDuration(time.Second) - return nil -} - -func (d ScannableDuration) Value() (driver.Value, error) { - return int64(time.Duration(d) / time.Second), nil -} diff --git a/models/config.go b/models/config.go index 80fe253..436814a 100644 --- a/models/config.go +++ b/models/config.go @@ -1,13 +1,10 @@ package models -import "time" - type Config struct { - Port int - DbHost string - DbUser string - DbPassword string - DbName string - DbDialect string - AggregationInterval time.Duration + Port int + DbHost string + DbUser string + DbPassword string + DbName string + DbDialect string } diff --git a/models/heartbeat.go b/models/heartbeat.go index 8070411..22c8a80 100644 --- a/models/heartbeat.go +++ b/models/heartbeat.go @@ -7,15 +7,13 @@ import ( "strconv" "strings" "time" - - "github.com/jinzhu/gorm" ) type HeartbeatReqTime time.Time type Heartbeat struct { - gorm.Model - User *User `json:"user" gorm:"not null; association_foreignkey:ID"` + ID uint `gorm:"primary_key"` + User *User `json:"-" gorm:"not null; index:idx_time_user"` UserID string `json:"-" gorm:"not null; index:idx_time_user"` Entity string `json:"entity" gorm:"not null"` Type string `json:"type"` diff --git a/models/summary.go b/models/summary.go new file mode 100644 index 0000000..5bf5c86 --- /dev/null +++ b/models/summary.go @@ -0,0 +1,28 @@ +package models + +import ( + "time" +) + +const ( + NSummaryTypes uint8 = 4 + SummaryProject uint8 = 0 + SummaryLanguage uint8 = 1 + SummaryEditor uint8 = 2 + SummaryOS uint8 = 3 +) + +type Summary struct { + UserID string `json:"user_id"` + FromTime *time.Time `json:"from"` + ToTime *time.Time `json:"to"` + Projects []SummaryItem `json:"projects"` + Languages []SummaryItem `json:"languages"` + Editors []SummaryItem `json:"editors"` + OperatingSystems []SummaryItem `json:"operating_systems"` +} + +type SummaryItem struct { + Key string `json:"key"` + Total time.Duration `json:"total"` +} diff --git a/routes/aggregation.go b/routes/aggregation.go deleted file mode 100644 index f80ca56..0000000 --- a/routes/aggregation.go +++ /dev/null @@ -1,49 +0,0 @@ -package routes - -import ( - "net/http" - "time" - - "github.com/n1try/wakapi/models" - "github.com/n1try/wakapi/services" - "github.com/n1try/wakapi/utils" -) - -type AggregationHandler struct { - AggregationSrvc *services.AggregationService -} - -func (h *AggregationHandler) Get(w http.ResponseWriter, r *http.Request) { - if r.Method != "GET" { - w.WriteHeader(415) - return - } - - user := r.Context().Value(models.UserKey).(*models.User) - params := r.URL.Query() - from, err := utils.ParseDate(params.Get("from")) - if err != nil { - w.WriteHeader(400) - w.Write([]byte("Missing 'from' parameter")) - return - } - - to, err := utils.ParseDate(params.Get("to")) - if err != nil { - to = time.Now() - } - - aggregations, err := h.AggregationSrvc.FindOrAggregate(from, to, user) - if err != nil { - w.WriteHeader(500) - return - } - for i := 0; i < len(aggregations); i++ { - if err := h.AggregationSrvc.SaveAggregation(aggregations[i]); err != nil { - w.WriteHeader(500) - return - } - } - - w.WriteHeader(200) -} diff --git a/routes/heartbeat.go b/routes/heartbeat.go index 02c3f57..5537b69 100644 --- a/routes/heartbeat.go +++ b/routes/heartbeat.go @@ -16,8 +16,8 @@ type HeartbeatHandler struct { } func (h *HeartbeatHandler) Post(w http.ResponseWriter, r *http.Request) { - if r.Method != "POST" { - w.WriteHeader(415) + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) return } @@ -27,7 +27,7 @@ func (h *HeartbeatHandler) Post(w http.ResponseWriter, r *http.Request) { dec := json.NewDecoder(r.Body) if err := dec.Decode(&heartbeats); err != nil { - w.WriteHeader(400) + w.WriteHeader(http.StatusBadRequest) w.Write([]byte(err.Error())) return } @@ -39,17 +39,17 @@ func (h *HeartbeatHandler) Post(w http.ResponseWriter, r *http.Request) { h.UserID = user.ID if !h.Valid() { - w.WriteHeader(400) + w.WriteHeader(http.StatusBadRequest) w.Write([]byte("Invalid heartbeat object.")) return } } if err := h.HeartbeatSrvc.InsertBatch(heartbeats); err != nil { - w.WriteHeader(500) + w.WriteHeader(http.StatusInternalServerError) os.Stderr.WriteString(err.Error()) return } - w.WriteHeader(200) + w.WriteHeader(http.StatusOK) } diff --git a/routes/summary.go b/routes/summary.go new file mode 100644 index 0000000..f45e34c --- /dev/null +++ b/routes/summary.go @@ -0,0 +1,41 @@ +package routes + +import ( + "net/http" + "time" + + "github.com/n1try/wakapi/models" + "github.com/n1try/wakapi/services" + "github.com/n1try/wakapi/utils" +) + +type SummaryHandler struct { + SummarySrvc *services.SummaryService +} + +func (h *SummaryHandler) Get(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + + user := r.Context().Value(models.UserKey).(*models.User) + params := r.URL.Query() + from, err := utils.ParseDate(params.Get("from")) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("Missing 'from' parameter")) + return + } + + now := time.Now() + to := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) // Start of current day + + summary, err := h.SummarySrvc.GetSummary(from, to, user) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + + utils.RespondJSON(w, http.StatusOK, summary) +} diff --git a/services/aggregation.go b/services/aggregation.go deleted file mode 100644 index 2ff12e3..0000000 --- a/services/aggregation.go +++ /dev/null @@ -1,142 +0,0 @@ -package services - -import ( - "log" - "math" - "time" - - "github.com/jinzhu/gorm" - "github.com/n1try/wakapi/models" -) - -type AggregationService struct { - Config *models.Config - Db *gorm.DB - HeartbeatService *HeartbeatService -} - -func (srv *AggregationService) SaveAggregation(aggregation *models.Aggregation) error { - if err := srv.Db.Save(aggregation).Error; err != nil { - return err - } - return nil -} - -func (srv *AggregationService) DeleteAggregations(from, to time.Time) error { - // TODO - return nil -} - -func (srv *AggregationService) FindOrAggregate(from, to time.Time, user *models.User) ([]*models.Aggregation, error) { - var existingAggregations []*models.Aggregation - if err := srv.Db. - Where(&models.Aggregation{UserID: user.ID}). - Where("from_time <= ?", from). - Where("to_time <= ?", to). - Order("to_time desc"). - Limit(models.NAggregationTypes). - Find(&existingAggregations).Error; err != nil { - return nil, err - } - - maxTo := getMaxTo(existingAggregations) - - if len(existingAggregations) == 0 { - newAggregations := srv.aggregate(from, to, user) - for i := 0; i < len(newAggregations); i++ { - srv.SaveAggregation(newAggregations[i]) - } - return newAggregations, nil - } else if maxTo.Before(to) { - // newAggregations := srv.aggregate(maxTo, to, user) - // TODO: compute aggregation(s) for remaining heartbeats - // TODO: if these aggregations are more than 24h, save them - // NOTE: never save aggregations that are less than 24h -> no need to delete some later - } else if maxTo.Equal(to) { - return existingAggregations, nil - } - - // Should never occur - return make([]*models.Aggregation, 0), nil -} - -func (srv *AggregationService) aggregate(from, to time.Time, user *models.User) []*models.Aggregation { - // TODO: Handle case that a time frame >= 24h is requested -> more than 4 will be returned - types := []uint8{models.AggregationProject, models.AggregationLanguage, models.AggregationEditor, models.AggregationOS} - heartbeats, err := srv.HeartbeatService.GetAllFrom(from, user) - if err != nil { - log.Fatal(err) - } - - var aggregations []*models.Aggregation - for _, t := range types { - aggregation := &models.Aggregation{ - UserID: user.ID, - FromTime: &from, - ToTime: &to, - Duration: to.Sub(from), - Type: t, - Items: srv.aggregateBy(heartbeats, t)[0:1], //make([]*models.AggregationItem, 0), - } - aggregations = append(aggregations, aggregation) - } - - return aggregations -} - -func (srv *AggregationService) aggregateBy(heartbeats []*models.Heartbeat, aggregationType uint8) []models.AggregationItem { - durations := make(map[string]time.Duration) - - for i, h := range heartbeats { - var key string - switch aggregationType { - case models.AggregationProject: - key = h.Project - case models.AggregationEditor: - key = h.Editor - case models.AggregationLanguage: - key = h.Language - case models.AggregationOS: - key = h.OperatingSystem - } - - if _, ok := durations[key]; !ok { - durations[key] = time.Duration(0) - } - - if i == 0 { - continue - } - - timePassed := h.Time.Time().Sub(heartbeats[i-1].Time.Time()) - timeThresholded := math.Min(float64(timePassed), float64(time.Duration(2)*time.Minute)) - durations[key] += time.Duration(int64(timeThresholded)) - } - - var items []models.AggregationItem - for k, v := range durations { - items = append(items, models.AggregationItem{ - AggregationID: 9, - Key: k, - Total: models.ScannableDuration(v), - }) - } - - return items -} - -func (srv *AggregationService) MergeAggregations(aggregations []*models.Aggregation) []*models.Aggregation { - // TODO - return make([]*models.Aggregation, 0) -} - -func getMaxTo(aggregations []*models.Aggregation) time.Time { - var maxTo time.Time - for i := 0; i < len(aggregations); i++ { - agg := aggregations[i] - if agg.ToTime.After(maxTo) { - maxTo = *agg.ToTime - } - } - return maxTo -} diff --git a/services/heartbeat.go b/services/heartbeat.go index 5f947df..92366b3 100644 --- a/services/heartbeat.go +++ b/services/heartbeat.go @@ -18,7 +18,7 @@ type HeartbeatService struct { func (srv *HeartbeatService) InsertBatch(heartbeats []*models.Heartbeat) error { var batch []interface{} for _, h := range heartbeats { - batch = append(batch, h) + batch = append(batch, *h) } if err := gormbulk.BulkInsert(srv.Db, batch, 3000); err != nil { @@ -27,12 +27,13 @@ func (srv *HeartbeatService) InsertBatch(heartbeats []*models.Heartbeat) error { return nil } -func (srv *HeartbeatService) GetAllFrom(date time.Time, user *models.User) ([]*models.Heartbeat, error) { +func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) { var heartbeats []*models.Heartbeat if err := srv.Db. Where(&models.Heartbeat{UserID: user.ID}). - Where("time > ?", date). - Find(heartbeats).Error; err != nil { + Where("time >= ?", from). + Where("time <= ?", to). + Find(&heartbeats).Error; err != nil { return nil, err } return heartbeats, nil diff --git a/services/summary.go b/services/summary.go new file mode 100644 index 0000000..71dd88c --- /dev/null +++ b/services/summary.go @@ -0,0 +1,74 @@ +package services + +import ( + "math" + "time" + + "github.com/jinzhu/gorm" + "github.com/n1try/wakapi/models" +) + +type SummaryService struct { + Config *models.Config + Db *gorm.DB + HeartbeatService *HeartbeatService +} + +func (srv *SummaryService) GetSummary(from, to time.Time, user *models.User) (*models.Summary, error) { + heartbeats, err := srv.HeartbeatService.GetAllWithin(from, to, user) + if err != nil { + return nil, err + } + + summary := &models.Summary{ + UserID: user.ID, + FromTime: &from, + ToTime: &to, + Projects: srv.aggregateBy(heartbeats, models.SummaryProject), + Languages: srv.aggregateBy(heartbeats, models.SummaryLanguage), + Editors: srv.aggregateBy(heartbeats, models.SummaryEditor), + OperatingSystems: srv.aggregateBy(heartbeats, models.SummaryOS), + } + + return summary, nil +} + +func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, aggregationType uint8) []models.SummaryItem { + durations := make(map[string]time.Duration) + + for i, h := range heartbeats { + var key string + switch aggregationType { + case models.SummaryProject: + key = h.Project + case models.SummaryEditor: + key = h.Editor + case models.SummaryLanguage: + key = h.Language + case models.SummaryOS: + key = h.OperatingSystem + } + + if _, ok := durations[key]; !ok { + durations[key] = time.Duration(0) + } + + if i == 0 { + continue + } + + timePassed := h.Time.Time().Sub(heartbeats[i-1].Time.Time()) + timeThresholded := math.Min(float64(timePassed), float64(time.Duration(2)*time.Minute)) + durations[key] += time.Duration(int64(timeThresholded)) + } + + items := make([]models.SummaryItem, 0) + for k, v := range durations { + items = append(items, models.SummaryItem{ + Key: k, + Total: v / time.Second, + }) + } + + return items +} diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 0000000..f5fcf04 --- /dev/null +++ b/utils/http.go @@ -0,0 +1,14 @@ +package utils + +import ( + "encoding/json" + "net/http" +) + +func RespondJSON(w http.ResponseWriter, status int, object interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(object); err != nil { + w.WriteHeader(http.StatusInternalServerError) + } +}