chore: make aggregation time configurable (resolve #60)

fix: doubly included heartbeats
fix: cross-day heartbeats are ignored for consistency

README.md

@@ -5,6 +5,11 @@
 ![](https://img.shields.io/github/license/muety/wakapi?style=flat-square)
 [![Go Report Card](https://goreportcard.com/badge/github.com/muety/wakapi?style=flat-square)](https://goreportcard.com/report/github.com/muety/wakapi)
 ![Coding Activity](https://img.shields.io/endpoint?url=https://apps.muetsch.io/wakapi/api/compat/shields/v1/n1try/interval:any/project:wakapi&style=flat-square&color=blue)
+
+[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=muety_wakapi&metric=security_rating)](https://sonarcloud.io/dashboard?id=muety_wakapi)
+[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=muety_wakapi&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=muety_wakapi)
+[![Technical Debt](https://sonarcloud.io/api/project_badges/measure?project=muety_wakapi&metric=sqale_index)](https://sonarcloud.io/dashboard?id=muety_wakapi)
+[![Lines of Code](https://sonarcloud.io/api/project_badges/measure?project=muety_wakapi&metric=ncloc)](https://sonarcloud.io/dashboard?id=muety_wakapi)
 ---
 
 **A minimalist, self-hosted WakaTime-compatible backend for coding statistics**

config.default.yml

@@ -7,6 +7,7 @@ server:
 
 app:
   cleanup: false            # only edit, if you know what you're doing
+  aggregation_time: '02:15' # time at which to run daily aggregation batch jobs
   custom_languages:
     vue: Vue
     jsx: JSX

config/config.go

@@ -27,6 +27,7 @@ var (
 
 type appConfig struct {
 	CleanUp         bool              `default:"false" env:"WAKAPI_CLEANUP"`
+	AggregationTime string            `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"`
 	CustomLanguages map[string]string `yaml:"custom_languages"`
 	LanguageColors  map[string]string `yaml:"-"`
 }

config/templates.go

@@ -0,0 +1,9 @@
+package config
+
+const (
+	IndexTemplate    = "index.tpl.html"
+	ImprintTemplate  = "imprint.tpl.html"
+	SignupTemplate   = "signup.tpl.html"
+	SettingsTemplate = "settings.tpl.html"
+	SummaryTemplate  = "summary.tpl.html"
+)

main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/gorilla/handlers"
 	conf "github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/migrations/common"
 	"log"
 	"net/http"
 	"strconv"
@@ -14,7 +15,6 @@ import (
 	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/muety/wakapi/middlewares"
-	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/routes"
 	shieldsV1Routes "github.com/muety/wakapi/routes/compat/shields/v1"
 	wtV1Routes "github.com/muety/wakapi/routes/compat/wakatime/v1"
@@ -64,12 +64,11 @@ func main() {
 		log.Println(err)
 		log.Fatal("could not connect to database")
 	}
-	// TODO: Graceful shutdown
 	defer db.Close()
 
 	// Migrate database schema
 	runDatabaseMigrations()
-	applyFixtures()
+	runCustomMigrations()
 
 	// Services
 	aliasService = services.NewAliasService(db)
@@ -79,9 +78,6 @@ func main() {
 	aggregationService = services.NewAggregationService(db, userService, summaryService, heartbeatService)
 	keyValueService = services.NewKeyValueService(db)
 
-	// Custom migrations and initial data
-	migrateLanguages()
-
 	// Aggregate heartbeats to summaries and persist them
 	go aggregationService.Schedule()
 
@@ -176,25 +172,9 @@ func runDatabaseMigrations() {
 	}
 }
 
-func applyFixtures() {
+func runCustomMigrations() {
-	if err := config.GetFixturesFunc(config.Db.Dialect)(db); err != nil {
+	common.ApplyFixtures(db)
-		log.Fatal(err)
+	common.MigrateLanguages(db)
-	}
-}
-
-func migrateLanguages() {
-	for k, v := range config.App.CustomLanguages {
-		result := db.Model(models.Heartbeat{}).
-			Where("language = ?", "").
-			Where("entity LIKE ?", "%."+k).
-			Updates(models.Heartbeat{Language: v})
-		if result.Error != nil {
-			log.Fatal(result.Error)
-		}
-		if result.RowsAffected > 0 {
-			log.Printf("Migrated %+v rows for custom language %+s.\n", result.RowsAffected, k)
-		}
-	}
 }
 
 func promptAbort(message string, timeoutSec int) {

middlewares/authenticate.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	config2 "github.com/muety/wakapi/config"
+	conf "github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/utils"
 	"log"
 	"net/http"
@@ -18,7 +18,7 @@ import (
 )
 
 type AuthenticateMiddleware struct {
-	config         *config2.Config
+	config         *conf.Config
 	userSrvc       *services.UserService
 	cache          *cache.Cache
 	whitelistPaths []string
@@ -26,7 +26,7 @@ type AuthenticateMiddleware struct {
 
 func NewAuthenticateMiddleware(userService *services.UserService, whitelistPaths []string) *AuthenticateMiddleware {
 	return &AuthenticateMiddleware{
-		config:         config2.Get(),
+		config:         conf.Get(),
 		userSrvc:       userService,
 		cache:          cache.New(1*time.Hour, 2*time.Hour),
 		whitelistPaths: whitelistPaths,
@@ -117,12 +117,12 @@ func (m *AuthenticateMiddleware) tryGetUserByCookie(r *http.Request) (*models.Us
 // migrate old md5-hashed passwords to new salted bcrypt hashes for backwards compatibility
 func CheckAndMigratePassword(user *models.User, login *models.Login, salt string, userServiceRef *services.UserService) bool {
 	if utils.IsMd5(user.Password) {
-		if utils.CheckPasswordMd5(user, login.Password) {
+		if utils.CompareMd5(user.Password, login.Password, "") {
 			log.Printf("migrating old md5 password to new bcrypt format for user '%s'", user.ID)
 			userServiceRef.MigrateMd5Password(user, login)
 			return true
 		}
 		return false
 	}
-	return utils.CheckPasswordBcrypt(user, login.Password, salt)
+	return utils.CompareBcrypt(user.Password, login.Password, salt)
 }

migrations/common/fixtures.go

@@ -0,0 +1,15 @@
+package common
+
+import (
+	"github.com/jinzhu/gorm"
+	"github.com/muety/wakapi/config"
+	"log"
+)
+
+func ApplyFixtures(db *gorm.DB) {
+	cfg := config.Get()
+
+	if err := cfg.GetFixturesFunc(cfg.Db.Dialect)(db); err != nil {
+		log.Fatal(err)
+	}
+}

migrations/common/languages.go

@@ -0,0 +1,25 @@
+package common
+
+import (
+	"github.com/jinzhu/gorm"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"log"
+)
+
+func MigrateLanguages(db *gorm.DB) {
+	cfg := config.Get()
+
+	for k, v := range cfg.App.CustomLanguages {
+		result := db.Model(models.Heartbeat{}).
+			Where("language = ?", "").
+			Where("entity LIKE ?", "%."+k).
+			Updates(models.Heartbeat{Language: v})
+		if result.Error != nil {
+			log.Fatal(result.Error)
+		}
+		if result.RowsAffected > 0 {
+			log.Printf("Migrated %+v rows for custom language %+s.\n", result.RowsAffected, k)
+		}
+	}
+}

models/compat/wakatime/v1/summaries.go

@@ -64,11 +64,11 @@ func NewSummariesFrom(summaries []*models.Summary, filters *models.Filters) *Sum
 	for i, s := range summaries {
 		data[i] = newDataFrom(s)
 
-		if s.FromTime.Before(minDate) {
+		if s.FromTime.T().Before(minDate) {
-			minDate = s.FromTime
+			minDate = s.FromTime.T()
 		}
-		if s.ToTime.After(maxDate) {
+		if s.ToTime.T().After(maxDate) {
-			maxDate = s.ToTime
+			maxDate = s.ToTime.T()
 		}
 	}
 
@@ -101,8 +101,8 @@ func newDataFrom(s *models.Summary) *summariesData {
 		},
 		Range: &summariesRange{
 			Date:     time.Now().Format(time.RFC3339),
-			End:      s.ToTime,
+			End:      s.ToTime.T(),
-			Start:    s.FromTime,
+			Start:    s.FromTime.T(),
 			Text:     "",
 			Timezone: zone,
 		},
@@ -158,13 +158,17 @@ func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *summariesEn
 	hrs := int(total.Hours())
 	mins := int((total - time.Duration(hrs)*time.Hour).Minutes())
 	secs := int((total - time.Duration(hrs)*time.Hour - time.Duration(mins)*time.Minute).Seconds())
+	percentage := math.Round((total.Seconds()/entityTotal.Seconds())*1e4) / 100
+	if math.IsNaN(percentage) || math.IsInf(percentage, 0) {
+		percentage = 0
+	}
 
 	return &summariesEntry{
 		Digital:      fmt.Sprintf("%d:%d:%d", hrs, mins, secs),
 		Hours:        hrs,
 		Minutes:      mins,
 		Name:         e.Key,
-		Percent:      math.Round((total.Seconds()/entityTotal.Seconds())*1e4) / 100,
+		Percent:      percentage,
 		Seconds:      secs,
 		Text:         utils.FmtWakatimeDuration(total),
 		TotalSeconds: total.Seconds(),

models/models.go

@@ -1,4 +1,5 @@
 package models
 
 func init() {
+	// nothing no init here, yet
 }

models/shared.go

@@ -76,28 +76,38 @@ func (j *CustomTime) UnmarshalJSON(b []byte) error {
 	return nil
 }
 
+// heartbeat timestamps arrive as strings for sqlite and as time.Time for postgres
 func (j *CustomTime) Scan(value interface{}) error {
+	var (
+		t   time.Time
+		err error
+	)
+
 	switch value.(type) {
 	case string:
-		t, err := time.Parse("2006-01-02 15:04:05-07:00", value.(string))
+		t, err = time.Parse("2006-01-02 15:04:05-07:00", value.(string))
 		if err != nil {
 			return errors.New(fmt.Sprintf("unsupported date time format: %s", value))
 		}
-		*j = CustomTime(t)
 	case int64:
-		*j = CustomTime(time.Unix(value.(int64), 0))
+		t = time.Unix(0, value.(int64))
 		break
 	case time.Time:
-		*j = CustomTime(value.(time.Time))
+		t = value.(time.Time)
 		break
 	default:
 		return errors.New(fmt.Sprintf("unsupported type: %T", value))
 	}
+
+	t = time.Unix(0, (t.UnixNano()/int64(time.Millisecond))*int64(time.Millisecond)) // round to millisecond precision
+	*j = CustomTime(t)
+
 	return nil
 }
 
 func (j CustomTime) Value() (driver.Value, error) {
-	return time.Time(j), nil
+	t := time.Unix(0, j.T().UnixNano()/int64(time.Millisecond)*int64(time.Millisecond)) // round to millisecond precision
+	return t, nil
 }
 
 func (j CustomTime) String() string {
@@ -105,6 +115,6 @@ func (j CustomTime) String() string {
 	return t.Format("2006-01-02 15:04:05.000")
 }
 
-func (j CustomTime) Time() time.Time {
+func (j CustomTime) T() time.Time {
 	return time.Time(j)
 }

models/summary.go

@@ -36,8 +36,8 @@ const UnknownSummaryKey = "unknown"
 type Summary struct {
 	ID               uint           `json:"-" gorm:"primary_key"`
 	UserID           string         `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
-	FromTime         time.Time      `json:"from" gorm:"not null; type:timestamp(3); default:CURRENT_TIMESTAMP(3); index:idx_time_summary_user"`
+	FromTime         CustomTime     `json:"from" gorm:"not null; type:timestamp(3); default:CURRENT_TIMESTAMP(3); index:idx_time_summary_user"`
-	ToTime           time.Time      `json:"to" gorm:"not null; type:timestamp(3); default:CURRENT_TIMESTAMP(3); index:idx_time_summary_user"`
+	ToTime           CustomTime     `json:"to" gorm:"not null; type:timestamp(3); default:CURRENT_TIMESTAMP(3); index:idx_time_summary_user"`
 	Projects         []*SummaryItem `json:"projects"`
 	Languages        []*SummaryItem `json:"languages"`
 	Editors          []*SummaryItem `json:"editors"`

routes/public.go

@@ -3,7 +3,7 @@ package routes
 import (
 	"fmt"
 	"github.com/gorilla/schema"
-	config2 "github.com/muety/wakapi/config"
+	conf "github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/middlewares"
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/services"
@@ -14,7 +14,7 @@ import (
 )
 
 type IndexHandler struct {
-	config       *config2.Config
+	config       *conf.Config
 	userSrvc     *services.UserService
 	keyValueSrvc *services.KeyValueService
 }
@@ -24,7 +24,7 @@ var signupDecoder = schema.NewDecoder()
 
 func NewIndexHandler(userService *services.UserService, keyValueService *services.KeyValueService) *IndexHandler {
 	return &IndexHandler{
-		config:       config2.Get(),
+		config:       conf.Get(),
 		userSrvc:     userService,
 		keyValueSrvc: keyValueService,
 	}
@@ -44,7 +44,7 @@ func (h *IndexHandler) GetIndex(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	templates["index.tpl.html"].Execute(w, nil)
+	templates[conf.IndexTemplate].Execute(w, nil)
 }
 
 func (h *IndexHandler) GetImprint(w http.ResponseWriter, r *http.Request) {
@@ -57,7 +57,7 @@ func (h *IndexHandler) GetImprint(w http.ResponseWriter, r *http.Request) {
 		text = data.Value
 	}
 
-	templates["imprint.tpl.html"].Execute(w, &struct {
+	templates[conf.ImprintTemplate].Execute(w, &struct {
 		HtmlText string
 	}{HtmlText: text})
 }
@@ -133,11 +133,11 @@ func (h *IndexHandler) GetSignup(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if handleAlerts(w, r, "signup.tpl.html") {
+	if handleAlerts(w, r, conf.SignupTemplate) {
 		return
 	}
 
-	templates["signup.tpl.html"].Execute(w, nil)
+	templates[conf.SignupTemplate].Execute(w, nil)
 }
 
 func (h *IndexHandler) PostSignup(w http.ResponseWriter, r *http.Request) {
@@ -152,26 +152,26 @@ func (h *IndexHandler) PostSignup(w http.ResponseWriter, r *http.Request) {
 
 	var signup models.Signup
 	if err := r.ParseForm(); err != nil {
-		respondAlert(w, "missing parameters", "", "signup.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "missing parameters", "", conf.SignupTemplate, http.StatusBadRequest)
 		return
 	}
 	if err := signupDecoder.Decode(&signup, r.PostForm); err != nil {
-		respondAlert(w, "missing parameters", "", "signup.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "missing parameters", "", conf.SignupTemplate, http.StatusBadRequest)
 		return
 	}
 
 	if !signup.IsValid() {
-		respondAlert(w, "invalid parameters", "", "signup.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "invalid parameters", "", conf.SignupTemplate, http.StatusBadRequest)
 		return
 	}
 
 	_, created, err := h.userSrvc.CreateOrGet(&signup)
 	if err != nil {
-		respondAlert(w, "failed to create new user", "", "signup.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "failed to create new user", "", conf.SignupTemplate, http.StatusInternalServerError)
 		return
 	}
 	if !created {
-		respondAlert(w, "user already existing", "", "signup.tpl.html", http.StatusConflict)
+		respondAlert(w, "user already existing", "", conf.SignupTemplate, http.StatusConflict)
 		return
 	}
 

routes/routes.go

@@ -59,7 +59,7 @@ func loadTemplates() {
 func respondAlert(w http.ResponseWriter, error, success, tplName string, status int) {
 	w.WriteHeader(status)
 	if tplName == "" {
-		tplName = "index.tpl.html"
+		tplName = config.IndexTemplate
 	}
 	templates[tplName].Execute(w, struct {
 		Error   string

routes/settings.go

@@ -3,7 +3,7 @@ package routes
 import (
 	"fmt"
 	"github.com/gorilla/schema"
-	config2 "github.com/muety/wakapi/config"
+	conf "github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/services"
 	"github.com/muety/wakapi/utils"
@@ -12,7 +12,7 @@ import (
 )
 
 type SettingsHandler struct {
-	config   *config2.Config
+	config   *conf.Config
 	userSrvc *services.UserService
 }
 
@@ -20,7 +20,7 @@ var credentialsDecoder = schema.NewDecoder()
 
 func NewSettingsHandler(userService *services.UserService) *SettingsHandler {
 	return &SettingsHandler{
-		config:   config2.Get(),
+		config:   conf.Get(),
 		userSrvc: userService,
 	}
 }
@@ -36,10 +36,10 @@ func (h *SettingsHandler) GetIndex(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// TODO: when alerts are present, other data will not be passed to the template
-	if handleAlerts(w, r, "settings.tpl.html") {
+	if handleAlerts(w, r, conf.SettingsTemplate) {
 		return
 	}
-	templates["settings.tpl.html"].Execute(w, data)
+	templates[conf.SettingsTemplate].Execute(w, data)
 }
 
 func (h *SettingsHandler) PostCredentials(w http.ResponseWriter, r *http.Request) {
@@ -51,32 +51,34 @@ func (h *SettingsHandler) PostCredentials(w http.ResponseWriter, r *http.Request
 
 	var credentials models.CredentialsReset
 	if err := r.ParseForm(); err != nil {
-		respondAlert(w, "missing parameters", "", "settings.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "missing parameters", "", conf.SettingsTemplate, http.StatusBadRequest)
 		return
 	}
 	if err := credentialsDecoder.Decode(&credentials, r.PostForm); err != nil {
-		respondAlert(w, "missing parameters", "", "settings.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "missing parameters", "", conf.SettingsTemplate, http.StatusBadRequest)
 		return
 	}
 
-	if !utils.CheckPasswordBcrypt(user, credentials.PasswordOld, h.config.Security.PasswordSalt) {
+	if !utils.CompareBcrypt(user.Password, credentials.PasswordOld, h.config.Security.PasswordSalt) {
-		respondAlert(w, "invalid credentials", "", "settings.tpl.html", http.StatusUnauthorized)
+		respondAlert(w, "invalid credentials", "", conf.SettingsTemplate, http.StatusUnauthorized)
 		return
 	}
 
 	if !credentials.IsValid() {
-		respondAlert(w, "invalid parameters", "", "settings.tpl.html", http.StatusBadRequest)
+		respondAlert(w, "invalid parameters", "", conf.SettingsTemplate, http.StatusBadRequest)
 		return
 	}
 
 	user.Password = credentials.PasswordNew
-	if err := utils.HashPassword(user, h.config.Security.PasswordSalt); err != nil {
+	if hash, err := utils.HashBcrypt(user.Password, h.config.Security.PasswordSalt); err != nil {
-		respondAlert(w, "internal server error", "", "settings.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "internal server error", "", conf.SettingsTemplate, http.StatusInternalServerError)
 		return
+	} else {
+		user.Password = hash
 	}
 
 	if _, err := h.userSrvc.Update(user); err != nil {
-		respondAlert(w, "internal server error", "", "settings.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "internal server error", "", conf.SettingsTemplate, http.StatusInternalServerError)
 		return
 	}
 
@@ -86,7 +88,7 @@ func (h *SettingsHandler) PostCredentials(w http.ResponseWriter, r *http.Request
 	}
 	encoded, err := h.config.Security.SecureCookie.Encode(models.AuthCookieKey, login)
 	if err != nil {
-		respondAlert(w, "internal server error", "", "settings.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "internal server error", "", conf.SettingsTemplate, http.StatusInternalServerError)
 		return
 	}
 
@@ -110,7 +112,7 @@ func (h *SettingsHandler) PostResetApiKey(w http.ResponseWriter, r *http.Request
 
 	user := r.Context().Value(models.UserKey).(*models.User)
 	if _, err := h.userSrvc.ResetApiKey(user); err != nil {
-		respondAlert(w, "internal server error", "", "settings.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "internal server error", "", conf.SettingsTemplate, http.StatusInternalServerError)
 		return
 	}
 
@@ -126,7 +128,7 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
 	user := r.Context().Value(models.UserKey).(*models.User)
 
 	if _, err := h.userSrvc.ToggleBadges(user); err != nil {
-		respondAlert(w, "internal server error", "", "settings.tpl.html", http.StatusInternalServerError)
+		respondAlert(w, "internal server error", "", conf.SettingsTemplate, http.StatusInternalServerError)
 		return
 	}
 

routes/summary.go

@@ -1,7 +1,7 @@
 package routes
 
 import (
-	config2 "github.com/muety/wakapi/config"
+	conf "github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/services"
 	"github.com/muety/wakapi/utils"
@@ -10,13 +10,13 @@ import (
 
 type SummaryHandler struct {
 	summarySrvc *services.SummaryService
-	config      *config2.Config
+	config      *conf.Config
 }
 
 func NewSummaryHandler(summaryService *services.SummaryService) *SummaryHandler {
 	return &SummaryHandler{
 		summarySrvc: summaryService,
-		config:      config2.Get(),
+		config:      conf.Get(),
 	}
 }
 
@@ -44,13 +44,13 @@ func (h *SummaryHandler) GetIndex(w http.ResponseWriter, r *http.Request) {
 
 	summary, err, status := h.loadUserSummary(r)
 	if err != nil {
-		respondAlert(w, err.Error(), "", "summary.tpl.html", status)
+		respondAlert(w, err.Error(), "", conf.SummaryTemplate, status)
 		return
 	}
 
 	user := r.Context().Value(models.UserKey).(*models.User)
 	if user == nil {
-		respondAlert(w, "unauthorized", "", "summary.tpl.html", http.StatusUnauthorized)
+		respondAlert(w, "unauthorized", "", conf.SummaryTemplate, http.StatusUnauthorized)
 		return
 	}
 
@@ -60,7 +60,7 @@ func (h *SummaryHandler) GetIndex(w http.ResponseWriter, r *http.Request) {
 		ApiKey:         user.ApiKey,
 	}
 
-	templates["summary.tpl.html"].Execute(w, vm)
+	templates[conf.SummaryTemplate].Execute(w, vm)
 }
 
 func (h *SummaryHandler) loadUserSummary(r *http.Request) (*models.Summary, error, int) {

scripts/push_sample_data.py

@@ -1,15 +1,17 @@
 #!/usr/bin/python3
+
+import argparse
+import base64
 import random
 import string
 import sys
 from datetime import datetime, timedelta
-from typing import List
+from typing import List, Union
 
 import requests
+from tqdm import tqdm
 
-N_PROJECTS = 5
+UA = 'wakatime/13.0.7 (Linux-4.15.0-91-generic-x86_64-with-glibc2.4) Python3.8.0.final.0 generator/1.42.1 generator-wakatime/4.0.0'
-N_PAST_HOURS = 24
-UA = 'wakatime/13.0.7 (Linux-4.15.0-91-generic-x86_64-with-glibc2.4) Python3.8.0.final.0 vscode/1.42.1 vscode-wakatime/4.0.0'
 LANGUAGES = {
     'Go': 'go',
     'Java': 'java',
@@ -36,23 +38,25 @@ class Heartbeat:
         self.is_write: bool = is_write
         self.branch: str = branch
         self.type: str = type
-        self.category: str = None
+        self.category: Union[str, None] = None
 
 
-def generate_data(n: int) -> List[Heartbeat]:
+def generate_data(n: int, n_projects: int = 5, n_past_hours: int = 24) -> List[Heartbeat]:
     data: List[Heartbeat] = []
     now: datetime = datetime.today()
-    projects: List[str] = [randomword(random.randint(5, 10)) for _ in range(5)]
+    projects: List[str] = [randomword(random.randint(5, 10)) for _ in range(n_projects)]
     languages: List[str] = list(LANGUAGES.keys())
 
-    for i in range(n):
+    for _ in range(n):
         p: str = random.choice(projects)
         l: str = random.choice(languages)
         f: str = randomword(random.randint(2, 8))
         delta: timedelta = timedelta(
-            hours=random.randint(0, N_PAST_HOURS - 1),
+            hours=random.randint(0, n_past_hours - 1),
             minutes=random.randint(0, 59),
-            seconds=random.randint(0, 59)
+            seconds=random.randint(0, 59),
+            milliseconds=random.randint(0, 999),
+            microseconds=random.randint(0, 999)
         )
 
         data.append(Heartbeat(
@@ -65,29 +69,43 @@ def generate_data(n: int) -> List[Heartbeat]:
     return data
 
 
-def post_data_sync(data: List[Heartbeat], url: str):
+def post_data_sync(data: List[Heartbeat], url: str, api_key: str):
-    for h in data:
+    encoded_key: str = str(base64.b64encode(api_key.encode('utf-8')), 'utf-8')
+
+    for h in tqdm(data):
         r = requests.post(url, json=[h.__dict__], headers={
-            'User-Agent': UA
+            'User-Agent': UA,
+            'Authorization': f'Basic {encoded_key}'
         })
-        if r.status_code != 200:
+        if r.status_code != 201:
             print(r.text)
             sys.exit(1)
 
 
 def randomword(length: int) -> str:
     letters = string.ascii_lowercase
-    return ''.join(random.choice(letters) for i in range(length))
+    return ''.join(random.choice(letters) for _ in range(length))
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Wakapi test data insertion script.')
+    parser.add_argument('-n', type=int, default=20, help='total number of random heartbeats to generate and insert')
+    parser.add_argument('-u', '--url', type=str, default='http://localhost:3000/api/heartbeat',
+                        help='url of your api\'s heartbeats endpoint')
+    parser.add_argument('-k', '--apikey', type=str, required=True,
+                        help='your api key (to get one, go to the web interface, create a new user, log in and copy the key)')
+    parser.add_argument('-p', '--projects', type=int, default=5, help='number of different fake projects to generate')
+    parser.add_argument('-o', '--offset', type=int, default=24,
+                        help='negative time offset in hours from now for to be used as an interval within which to generate heartbeats for')
+    parser.add_argument('-s', '--seed', type=int, default=2020,
+                        help='a seed for initializing the pseudo-random number generator')
+    return parser.parse_args()
 
 
 if __name__ == '__main__':
-    n: int = 10
+    args = parse_arguments()
-    url: str = 'http://admin:admin@localhost:3000/api/heartbeat'
 
-    if len(sys.argv) > 1:
+    random.seed(args.seed)
-        n = int(sys.argv[1])
-    if len(sys.argv) > 2:
-        url = sys.argv[2]
 
-    data: List[Heartbeat] = generate_data(n)
+    data: List[Heartbeat] = generate_data(args.n, args.projects, args.offset)
-    post_data_sync(data, url)
+    post_data_sync(data, args.url, args.apikey)

services/aggregation.go

@@ -12,7 +12,7 @@ import (
 )
 
 const (
-	aggregateIntervalDays int = 1 // TODO: Make configurable
+	aggregateIntervalDays int = 1
 )
 
 type AggregationService struct {
@@ -40,7 +40,6 @@ type AggregationJob struct {
 }
 
 // Schedule a job to (re-)generate summaries every day shortly after midnight
-// TODO: Make configurable
 func (srv *AggregationService) Schedule() {
 	jobs := make(chan *AggregationJob)
 	summaries := make(chan *models.Summary)
@@ -58,7 +57,7 @@ func (srv *AggregationService) Schedule() {
 	// Run once initially
 	srv.trigger(jobs)
 
-	gocron.Every(1).Day().At("02:15").Do(srv.trigger, jobs)
+	gocron.Every(1).Day().At(srv.Config.App.AggregationTime).Do(srv.trigger, jobs)
 	<-gocron.Start()
 }
 
@@ -98,7 +97,7 @@ func (srv *AggregationService) trigger(jobs chan<- *AggregationJob) error {
 
 	userSummaryTimes := make(map[string]time.Time)
 	for _, s := range latestSummaries {
-		userSummaryTimes[s.UserID] = s.ToTime
+		userSummaryTimes[s.UserID] = s.ToTime.T()
 	}
 
 	missingUserIDs := make([]string, 0)

services/heartbeat.go

@@ -46,7 +46,7 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
 	if err := srv.Db.
 		Where(&models.Heartbeat{UserID: user.ID}).
 		Where("time >= ?", from).
-		Where("time <= ?", to).
+		Where("time < ?", to).
 		Order("time asc").
 		Find(&heartbeats).Error; err != nil {
 		return nil, err

services/summary.go

@@ -14,6 +14,8 @@ import (
 	"github.com/muety/wakapi/models"
 )
 
+const HeartbeatDiffThreshold = 2 * time.Minute
+
 type SummaryService struct {
 	Config           *config.Config
 	Cache            *cache.Cache
@@ -37,6 +39,7 @@ type Interval struct {
 	End   time.Time
 }
 
+// TODO: simplify!
 func (srv *SummaryService) Construct(from, to time.Time, user *models.User, recompute bool) (*models.Summary, error) {
 	var existingSummaries []*models.Summary
 	var cacheKey string
@@ -102,8 +105,8 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco
 
 	realFrom, realTo := from, to
 	if len(existingSummaries) > 0 {
-		realFrom = existingSummaries[0].FromTime
+		realFrom = existingSummaries[0].FromTime.T()
-		realTo = existingSummaries[len(existingSummaries)-1].ToTime
+		realTo = existingSummaries[len(existingSummaries)-1].ToTime.T()
 
 		for _, summary := range existingSummaries {
 			summary.FillUnknown()
@@ -121,8 +124,8 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco
 
 	aggregatedSummary := &models.Summary{
 		UserID:           user.ID,
-		FromTime:         realFrom,
+		FromTime:         models.CustomTime(realFrom),
-		ToTime:           realTo,
+		ToTime:           models.CustomTime(realTo),
 		Projects:         projectItems,
 		Languages:        languageItems,
 		Editors:          editorItems,
@@ -217,9 +220,16 @@ func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryTy
 			continue
 		}
 
-		timePassed := h.Time.Time().Sub(heartbeats[i-1].Time.Time())
+		t1, t2, tdiff := h.Time.T(), heartbeats[i-1].Time.T(), time.Duration(0)
-		timeThresholded := math.Min(float64(timePassed), float64(time.Duration(2)*time.Minute))
+		// This is a hack. The time difference between two heartbeats from two subsequent day (e.g. 23:59:59 and 00:00:01) are ignored.
-		durations[key] += time.Duration(int64(timeThresholded))
+		// This is to prevent a discrepancy between summaries computed solely from heartbeats and summaries involving pre-aggregated per-day summaries.
+		// For the latter, a duration is already pre-computed and information about individual heartbeats is lost, so there can be no cross-day overflow.
+		// Essentially, we simply ignore such edge-case heartbeats here, which makes the eventual total duration potentially a bit shorter.
+		if t1.Day() == t2.Day() {
+			timePassed := t1.Sub(t2)
+			tdiff = time.Duration(int64(math.Min(float64(timePassed), float64(HeartbeatDiffThreshold))))
+		}
+		durations[key] += tdiff
 	}
 
 	items := make([]*models.SummaryItem, 0)
@@ -246,13 +256,13 @@ func getMissingIntervals(from, to time.Time, existingSummaries []*models.Summary
 	intervals := make([]*Interval, 0)
 
 	// Pre
-	if from.Before(existingSummaries[0].FromTime) {
+	if from.Before(existingSummaries[0].FromTime.T()) {
-		intervals = append(intervals, &Interval{from, existingSummaries[0].FromTime})
+		intervals = append(intervals, &Interval{from, existingSummaries[0].FromTime.T()})
 	}
 
 	// Between
 	for i := 0; i < len(existingSummaries)-1; i++ {
-		t1, t2 := existingSummaries[i].ToTime, existingSummaries[i+1].FromTime
+		t1, t2 := existingSummaries[i].ToTime.T(), existingSummaries[i+1].FromTime.T()
 		if t1.Equal(t2) {
 			continue
 		}
@@ -262,13 +272,13 @@ func getMissingIntervals(from, to time.Time, existingSummaries []*models.Summary
 		td2 := time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, t2.Location())
 		// one or more day missing in between?
 		if td1.Before(td2) {
-			intervals = append(intervals, &Interval{existingSummaries[i].ToTime, existingSummaries[i+1].FromTime})
+			intervals = append(intervals, &Interval{existingSummaries[i].ToTime.T(), existingSummaries[i+1].FromTime.T()})
 		}
 	}
 
 	// Post
-	if to.After(existingSummaries[len(existingSummaries)-1].ToTime) {
+	if to.After(existingSummaries[len(existingSummaries)-1].ToTime.T()) {
-		intervals = append(intervals, &Interval{to, existingSummaries[len(existingSummaries)-1].ToTime})
+		intervals = append(intervals, &Interval{existingSummaries[len(existingSummaries)-1].ToTime.T(), to})
 	}
 
 	return intervals
@@ -296,12 +306,12 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
 			return nil, errors.New("users don't match")
 		}
 
-		if s.FromTime.Before(minTime) {
+		if s.FromTime.T().Before(minTime) {
-			minTime = s.FromTime
+			minTime = s.FromTime.T()
 		}
 
-		if s.ToTime.After(maxTime) {
+		if s.ToTime.T().After(maxTime) {
-			maxTime = s.ToTime
+			maxTime = s.ToTime.T()
 		}
 
 		finalSummary.Projects = mergeSummaryItems(finalSummary.Projects, s.Projects)
@@ -311,8 +321,8 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
 		finalSummary.Machines = mergeSummaryItems(finalSummary.Machines, s.Machines)
 	}
 
-	finalSummary.FromTime = minTime
+	finalSummary.FromTime = models.CustomTime(minTime)
-	finalSummary.ToTime = maxTime
+	finalSummary.ToTime = models.CustomTime(maxTime)
 
 	return finalSummary, nil
 }

services/user.go

@@ -54,8 +54,10 @@ func (srv *UserService) CreateOrGet(signup *models.Signup) (*models.User, bool,
 		Password: signup.Password,
 	}
 
-	if err := utils.HashPassword(u, srv.Config.Security.PasswordSalt); err != nil {
+	if hash, err := utils.HashBcrypt(u.Password, srv.Config.Security.PasswordSalt); err != nil {
 		return nil, false, err
+	} else {
+		u.Password = hash
 	}
 
 	result := srv.Db.FirstOrCreate(u, &models.User{ID: u.ID})
@@ -103,8 +105,10 @@ func (srv *UserService) ToggleBadges(user *models.User) (*models.User, error) {
 
 func (srv *UserService) MigrateMd5Password(user *models.User, login *models.Login) (*models.User, error) {
 	user.Password = login.Password
-	if err := utils.HashPassword(user, srv.Config.Security.PasswordSalt); err != nil {
+	if hash, err := utils.HashBcrypt(user.Password, srv.Config.Security.PasswordSalt); err != nil {
 		return nil, err
+	} else {
+		user.Password = hash
 	}
 
 	result := srv.Db.Model(user).Update("password", user.Password)

static/assets/app.js

@@ -242,8 +242,8 @@ function equalizeHeights() {
     })
 }
 
-function getTotal(data) {
+function getTotal(items) {
-    let total = data.reduce((acc, d) => acc + d.total, 0)
+    let total = items.reduce((acc, d) => acc + d.total, 0)
     document.getElementById('total-span').innerText = total.toString().toHHMMSS()
 }
 

utils/auth.go

@@ -63,28 +63,29 @@ func IsMd5(hash string) bool {
 	return md5Regex.Match([]byte(hash))
 }
 
-func CheckPasswordBcrypt(user *models.User, password, salt string) bool {
+func CompareBcrypt(wanted, actual, pepper string) bool {
-	plainPassword := []byte(strings.TrimSpace(password) + salt)
+	plainPassword := []byte(strings.TrimSpace(actual) + pepper)
-	err := bcrypt.CompareHashAndPassword([]byte(user.Password), plainPassword)
+	err := bcrypt.CompareHashAndPassword([]byte(wanted), plainPassword)
 	return err == nil
 }
 
 // deprecated, only here for backwards compatibility
-func CheckPasswordMd5(user *models.User, password string) bool {
+func CompareMd5(wanted, actual, pepper string) bool {
-	hash := md5.Sum([]byte(password))
+	return HashMd5(actual, pepper) == wanted
-	hashStr := hex.EncodeToString(hash[:])
-	if hashStr == user.Password {
-		return true
-	}
-	return false
 }
 
-// inplace
+func HashBcrypt(plain, pepper string) (string, error) {
-func HashPassword(u *models.User, salt string) error {
+	plainPepperedPassword := []byte(strings.TrimSpace(plain) + pepper)
-	plainSaltedPassword := []byte(strings.TrimSpace(u.Password) + salt)
+	bytes, err := bcrypt.GenerateFromPassword(plainPepperedPassword, bcrypt.DefaultCost)
-	bytes, err := bcrypt.GenerateFromPassword(plainSaltedPassword, bcrypt.DefaultCost)
 	if err == nil {
-		u.Password = string(bytes)
+		return string(bytes), nil
 	}
-	return err
+	return "", err
+}
+
+func HashMd5(plain, pepper string) string {
+	plainPepperedPassword := []byte(strings.TrimSpace(plain) + pepper)
+	hash := md5.Sum(plainPepperedPassword)
+	hashStr := hex.EncodeToString(hash[:])
+	return hashStr
 }

utils/http.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"encoding/json"
+	"log"
 	"net/http"
 )
 
@@ -9,7 +10,7 @@ func RespondJSON(w http.ResponseWriter, status int, object interface{}) {
 	w.Header().Set("Content-Type", "application/json")
 	w.WriteHeader(status)
 	if err := json.NewEncoder(w).Encode(object); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
+		log.Printf("error while writing json response: %v", err)
 	}
 }
 

version.txt

@@ -1 +1 @@
-1.12.2
+1.12.5

views/imprint.tpl.html

@@ -1,4 +1,5 @@
-<html>
+<!DOCTYPE html>
+<html lang="en">
 
 {{ template "head.tpl.html" . }}
 

views/index.tpl.html

@@ -1,4 +1,5 @@
-<html>
+<!DOCTYPE html>
+<html lang="en">
 
 {{ template "head.tpl.html" . }}
 

views/settings.tpl.html

@@ -1,4 +1,5 @@
-<html>
+<!DOCTYPE html>
+<html lang="en">
 
 {{ template "head.tpl.html" . }}
 
@@ -86,7 +87,7 @@
                         <div class="flex flex-col mb-4">
                             <div class="flex justify-between my-2">
                                 <div>
-                                    <img class="with-url-src" src="https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:today&style=flat-square&color=blue&label=today"/>
+                                    <img class="with-url-src" src="https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:today&style=flat-square&color=blue&label=today" alt="Shields.io badge"/>
                                 </div>
                                 <span class="with-url-inner text-xs bg-gray-900 rounded py-1 px-2 font-mono whitespace-no-wrap overflow-auto" style="max-width: 300px;">
                                     https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:today&style=flat-square&color=blue&label=today
@@ -94,7 +95,7 @@
                             </div>
                             <div class="flex justify-between my-2">
                                 <div>
-                                    <img class="with-url-src" src="https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:30_days&style=flat-square&color=blue&label=last 30d"/>
+                                    <img class="with-url-src" src="https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:30_days&style=flat-square&color=blue&label=last 30d" alt="Shields.io badge"/>
                                 </div>
                                 <span class="with-url-inner text-xs bg-gray-900 rounded py-1 px-2 font-mono whitespace-no-wrap overflow-auto" style="max-width: 300px;">
                                     https://img.shields.io/endpoint?url=%s/api/compat/shields/v1/{{ .User.ID }}/interval:30_days&style=flat-square&color=blue&label=last 30d
@@ -104,7 +105,7 @@
 
                         <p>You can also add <span class="text-xs bg-gray-900 rounded py-1 px-2 font-mono">/project:your-cool-project</span> to the URL to filter by project.</p>
                     {{ else }}
-                        <p>You have the ability to create badges from your coding statistics using <a href="https://shields.io" target="_blank" class="border-b border-green-800">Shields.io</a>. To do so, you need to grant public, unauthorized access to the respective endpoint.</p>
+                        <p>You have the ability to create badges from your coding statistics using <a href="https://shields.io" target="_blank" class="border-b border-green-800" rel="noopener noreferrer">Shields.io</a>. To do so, you need to grant public, unauthorized access to the respective endpoint.</p>
                         <div class="flex justify-around mt-4">
                             <span class="font-mono font-normal bg-gray-900 p-1 rounded whitespace-no-wrap">GET /api/compat/shields/v1</span>
                             <button type="submit" class="py-1 px-2 rounded bg-green-700 hover:bg-green-800 text-white text-xs" title="Make endpoint public to enable badges">

views/signup.tpl.html

@@ -1,4 +1,5 @@
-<html>
+<!DOCTYPE html>
+<html lang="en">
 
 {{ template "head.tpl.html" . }}
 
@@ -19,9 +20,11 @@
             <p class="text-sm text-gray-300">
                 💡 In order to use Wakapi, you need to create an account.
                 After successful signup, you still need to set up the <a href="https://wakatime.com" target="_blank"
+                                                                         rel="noopener noreferrer"
                                                                          class="border-b border-green-700">WakaTime</a>
                 client tools.
                 Please refer to <a href="https://github.com/muety/wakapi#client-setup" target="_blank"
+                                   rel="noopener noreferrer"
                                    class="border-b border-green-700">this readme section</a> for instructions.
                 You will be able to view you <strong>API Key</strong> once you log in.
             </p>

views/summary.tpl.html

@@ -1,4 +1,5 @@
-<html>
+<!DOCTYPE html>
+<html lang="en">
 
 {{ template "head.tpl.html" . }}
 
@@ -54,8 +55,8 @@
     <div class="flex justify-center">
         <div class="p-1">
             <div class="flex justify-center p-4 bg-white rounded shadow">
-                <p class="mx-2"><strong>▶️</strong> <span title="Start Time">{{ .FromTime | date }}</span></p>
+                <p class="mx-2"><strong>▶️</strong> <span title="Start Time">{{ .FromTime.T | date }}</span></p>
-                <p class="mx-2"><strong>⏹</strong> <span title="End Time">{{ .ToTime | date }}</span></p>
+                <p class="mx-2"><strong>⏹</strong> <span title="End Time">{{ .ToTime.T | date }}</span></p>
                 <p class="mx-2"><strong>⏱</strong> <span id="total-span" title="Total Hours"></span></p>
             </div>
         </div>
@@ -69,7 +70,7 @@
                 </div>
                 <canvas id="chart-projects"></canvas>
                 <div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
-                    <img src="assets/images/no_data.svg" class="w-20"/>
+                    <img src="assets/images/no_data.svg" class="w-20" alt="No data"/>
                     <span class="text-sm mt-4">No data available ...</span>
                 </div>
             </div>
@@ -82,7 +83,7 @@
                 </div>
                 <canvas id="chart-os"></canvas>
                 <div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
-                    <img src="assets/images/no_data.svg" class="w-20"/>
+                    <img src="assets/images/no_data.svg" class="w-20" alt="No data"/>
                     <span class="text-sm mt-4">No data available ...</span>
                 </div>
             </div>
@@ -95,7 +96,7 @@
                 </div>
                 <canvas id="chart-language"></canvas>
                 <div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
-                    <img src="assets/images/no_data.svg" class="w-20"/>
+                    <img src="assets/images/no_data.svg" class="w-20" alt="No data"/>
                     <span class="text-sm mt-4">No data available ...</span>
                 </div>
             </div>
@@ -108,7 +109,7 @@
                 </div>
                 <canvas id="chart-editor"></canvas>
                 <div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
-                    <img src="assets/images/no_data.svg" class="w-20"/>
+                    <img src="assets/images/no_data.svg" class="w-20" alt="No data"/>
                     <span class="text-sm mt-4">No data available ...</span>
                 </div>
             </div>
@@ -121,7 +122,7 @@
                 </div>
                 <canvas id="chart-machine"></canvas>
                 <div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
-                    <img src="assets/images/no_data.svg" class="w-20"/>
+                    <img src="assets/images/no_data.svg" class="w-20" alt="No data"/>
                     <span class="text-sm mt-4">No data available ...</span>
                 </div>
             </div>