chore: metrics performance improvements

README.md

@@ -128,6 +128,11 @@ You can specify configuration options either via a config file (default: `config
 | YAML Key / Env. Variable                                                     | Default                                          | Description                                                                                                                                                              |
 |------------------------------------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `env` /<br>`ENVIRONMENT`                                                     | `dev`                                            | Whether to use development- or production settings                                                                                                                       |
+| `app.aggregation_time`<br>`WAKAPI_AGGREGATION_TIME`                          | `02:15`                                          | Time of day at which to periodically run summary generation for all users                                                                                                |
+| `app.report_time_weekly`<br>`WAKAPI_REPORT_TIME_WEEKLY`                      | `fri,18:00`                                      | Week day and time at which to send e-mail reports                                                                                                                        |
+| `app.import_batch_size`<br>`WAKAPI_IMPORT_BATCH_SIZE`                        | `50`                                             | Size of batches of heartbeats to insert to the database during importing from external services                                                                          |
+| `app.inactive_days`<br>`WAKAPI_INACTIVE_DAYS`                                | `7`                                              | Number of days after which to consider a user inactive (only for metrics)                                                                                                |
+| `app.heartbeat_max_age`<br>`WAKAPI_HEARTBEAT_MAX_AGE`                        | `4320h`                                          | Maximum acceptable age of a heartbeat (see [`ParseDuration`](https://pkg.go.dev/time#ParseDuration))                                                                     |
 | `app.custom_languages`                                                       | -                                                | Map from file endings to language names                                                                                                                                  |
 | `app.avatar_url_template`                                                    | (see [`config.default.yml`](config.default.yml)) | URL template for external user avatar images (e.g. from [Dicebear](https://dicebear.com) or [Gravatar](https://gravatar.com))                                            |
 | `server.port` /<br> `WAKAPI_PORT`                                            | `3000`                                           | Port to listen on                                                                                                                                                        |

config.default.yml

@@ -16,6 +16,7 @@ app:
   report_time_weekly: 'fri,18:00'     # time at which to fan out weekly reports (format: '<weekday)>,<daytime>')
   inactive_days: 7                    # time of previous days within a user must have logged in to be considered active
   import_batch_size: 50               # maximum number of heartbeats to insert into the database within one transaction
+  heartbeat_max_age: '4320h'          # maximum acceptable age of a heartbeat (see https://pkg.go.dev/time#ParseDuration)
   custom_languages:
     vue: Vue
     jsx: JSX

config/config.go

@@ -68,6 +68,7 @@ type appConfig struct {
 	ImportBackoffMin  int                          `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"`
 	ImportBatchSize   int                          `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"`
 	InactiveDays      int                          `yaml:"inactive_days" default:"7" env:"WAKAPI_INACTIVE_DAYS"`
+	HeartbeatMaxAge   string                       `yaml:"heartbeat_max_age" default:"4320h" env:"WAKAPI_HEARTBEAT_MAX_AGE"`
 	CountCacheTTLMin  int                          `yaml:"count_cache_ttl_min" default:"30" env:"WAKAPI_COUNT_CACHE_TTL_MIN"`
 	AvatarURLTemplate string                       `yaml:"avatar_url_template" default:"api/avatar/{username_hash}.svg"`
 	CustomLanguages   map[string]string            `yaml:"custom_languages"`
@@ -242,6 +243,11 @@ func (c *appConfig) GetWeeklyReportTime() string {
 	return strings.Split(c.ReportTimeWeekly, ",")[1]
 }
 
+func (c *appConfig) HeartbeatsMaxAge() time.Duration {
+	d, _ := time.ParseDuration(c.HeartbeatMaxAge)
+	return d
+}
+
 func (c *dbConfig) IsSQLite() bool {
 	return c.Dialect == "sqlite3"
 }
@@ -400,6 +406,9 @@ func Load(version string) *Config {
 	if _, err := time.Parse("15:04", config.App.AggregationTime); err != nil {
 		logbuch.Fatal("invalid interval set for aggregation_time")
 	}
+	if _, err := time.ParseDuration(config.App.HeartbeatMaxAge); err != nil {
+		logbuch.Fatal("invalid duration set for heartbeat_max_age")
+	}
 
 	Set(config)
 	return Get()

main.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/lpar/gzipped/v2"
-	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/routes/relay"
 
 	"github.com/emvi/logbuch"
@@ -58,6 +57,7 @@ var (
 	summaryRepository         repositories.ISummaryRepository
 	keyValueRepository        repositories.IKeyValueRepository
 	diagnosticsRepository     repositories.IDiagnosticsRepository
+	metricsRepository         *repositories.MetricsRepository
 )
 
 var (
@@ -149,6 +149,7 @@ func main() {
 	summaryRepository = repositories.NewSummaryRepository(db)
 	keyValueRepository = repositories.NewKeyValueRepository(db)
 	diagnosticsRepository = repositories.NewDiagnosticsRepository(db)
+	metricsRepository = repositories.NewMetricsRepository(db)
 
 	// Services
 	mailService = mail.NewMailService()
@@ -178,7 +179,7 @@ func main() {
 	healthApiHandler := api.NewHealthApiHandler(db)
 	heartbeatApiHandler := api.NewHeartbeatApiHandler(userService, heartbeatService, languageMappingService)
 	summaryApiHandler := api.NewSummaryApiHandler(userService, summaryService)
-	metricsHandler := api.NewMetricsHandler(userService, summaryService, heartbeatService, keyValueService)
+	metricsHandler := api.NewMetricsHandler(userService, summaryService, heartbeatService, keyValueService, metricsRepository)
 	diagnosticsHandler := api.NewDiagnosticsApiHandler(userService, diagnosticsService)
 	avatarHandler := api.NewAvatarHandler()
 
@@ -267,18 +268,6 @@ func main() {
 		middlewares.NewFileTypeFilterMiddleware([]string{".go"})(staticFileServer),
 	)
 
-	// Miscellaneous
-	// Pre-warm projects cache
-	if !config.IsDev() {
-		allUsers, err := userService.GetAll()
-		if err == nil {
-			logbuch.Info("pre-warming user project cache")
-			for _, u := range allUsers {
-				go heartbeatService.GetEntitySetByUser(models.SummaryProject, u)
-			}
-		}
-	}
-
 	// Listen HTTP
 	listen(router)
 }

migrations/20220317_align_num_heartbeats.go

@@ -0,0 +1,56 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220317-align_num_heartbeats"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			logbuch.Info("this may take a while!")
+
+			// find all summaries whose num_heartbeats is zero even though they have items
+			var faultyIds []uint
+
+			if err := db.Model(&models.Summary{}).
+				Distinct("summaries.id").
+				Joins("INNER JOIN summary_items ON summaries.num_heartbeats = 0 AND summaries.id = summary_items.summary_id").
+				Scan(&faultyIds).Error; err != nil {
+				return err
+			}
+
+			// update their heartbeats counter
+			result := db.
+				Table("summaries AS s1").
+				Where("s1.id IN ?", faultyIds).
+				Update(
+					"num_heartbeats",
+					db.
+						Model(&models.Heartbeat{}).
+						Select("COUNT(*)").
+						Where("user_id = ?", gorm.Expr("s1.user_id")).
+						Where("time BETWEEN ? AND ?", gorm.Expr("s1.from_time"), gorm.Expr("s1.to_time")),
+				)
+
+			if err := result.Error; err != nil {
+				return err
+			}
+
+			logbuch.Info("corrected heartbeats counter of %d summaries", result.RowsAffected)
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

migrations/20220318_mysql_timestamp_precision.go

@@ -0,0 +1,41 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220318-mysql_timestamp_precision"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			if cfg.Db.IsMySQL() {
+				logbuch.Info("altering heartbeats table, this may take a while (up to hours)")
+
+				db.Exec("SET foreign_key_checks=0;")
+				db.Exec("SET unique_checks=0;")
+				if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `time` TIMESTAMP(3) NOT NULL").Error; err != nil {
+					return err
+				}
+				if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `created_at` TIMESTAMP(3) NOT NULL").Error; err != nil {
+					return err
+				}
+				db.Exec("SET foreign_key_checks=1;")
+				db.Exec("SET unique_checks=1;")
+
+				logbuch.Info("migrated timestamp columns to millisecond precision")
+			}
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

migrations/202203191_drop_diagnostics_user.go

@@ -0,0 +1,39 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "202203191-drop_diagnostics_user"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			migrator := db.Migrator()
+
+			if migrator.HasColumn(&models.Diagnostics{}, "user_id") {
+				logbuch.Info("running migration '%s'", name)
+
+				if err := migrator.DropConstraint(&models.Diagnostics{}, "fk_diagnostics_user"); err != nil {
+					logbuch.Warn("failed to drop 'fk_diagnostics_user' constraint (%v)", err)
+				}
+
+				if err := migrator.DropColumn(&models.Diagnostics{}, "user_id"); err != nil {
+					logbuch.Warn("failed to drop user_id column of diagnostics (%v)", err)
+				}
+			}
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

migrations/20220319_add_user_project_idx.go

@@ -0,0 +1,35 @@
+package migrations
+
+import (
+	"fmt"
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220319-add_user_project_idx"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			idxName := "idx_user_project"
+
+			if !db.Migrator().HasIndex(&models.Heartbeat{}, idxName) {
+				logbuch.Info("running migration '%s'", name)
+				if err := db.Exec(fmt.Sprintf("create index %s on heartbeats (user_id, project)", idxName)).Error; err != nil {
+					logbuch.Warn("failed to create %s", idxName)
+				}
+			}
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

mocks/heartbeat_service.go

@@ -20,8 +20,8 @@ func (m *HeartbeatServiceMock) InsertBatch(heartbeats []*models.Heartbeat) error
 	return args.Error(0)
 }
 
-func (m *HeartbeatServiceMock) Count() (int64, error) {
-	args := m.Called()
+func (m *HeartbeatServiceMock) Count(a bool) (int64, error) {
+	args := m.Called(a)
 	return int64(args.Int(0)), args.Error(1)
 }
 
@@ -69,3 +69,8 @@ func (m *HeartbeatServiceMock) DeleteBefore(time time.Time) error {
 	args := m.Called(time)
 	return args.Error(0)
 }
+
+func (m *HeartbeatServiceMock) DeleteByUser(u *models.User) error {
+	args := m.Called(u)
+	return args.Error(0)
+}

models/diagnostics.go

@@ -2,8 +2,6 @@ package models
 
 type Diagnostics struct {
 	ID           uint   `gorm:"primary_key"`
-	User         *User  `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
-	UserID       string `json:"-" gorm:"not null; index:idx_diagnostics_user"`
 	Platform     string `json:"platform"`
 	Architecture string `json:"architecture"`
 	Plugin       string `json:"plugin"`

models/heartbeat.go

@@ -11,11 +11,11 @@ import (
 type Heartbeat struct {
 	ID              uint64     `gorm:"primary_key" hash:"ignore"`
 	User            *User      `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" hash:"ignore"`
-	UserID          string     `json:"-" gorm:"not null; index:idx_time_user"`
+	UserID          string     `json:"-" gorm:"not null; index:idx_time_user,idx_user_project"` // idx_user_project is for quickly fetching a user's project list (settings page)
 	Entity          string     `json:"entity" gorm:"not null"`
 	Type            string     `json:"type"`
 	Category        string     `json:"category"`
-	Project         string     `json:"project" gorm:"index:idx_project"`
+	Project         string     `json:"project" gorm:"index:idx_project,idx_user_project"`
 	Branch          string     `json:"branch" gorm:"index:idx_branch"`
 	Language        string     `json:"language" gorm:"index:idx_language"`
 	IsWrite         bool       `json:"is_write"`
@@ -23,17 +23,22 @@ type Heartbeat struct {
 	OperatingSystem string     `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
 	Machine         string     `json:"machine" gorm:"index:idx_machine" hash:"ignore"`                   // ignored because wakatime api doesn't return machines currently
 	UserAgent       string     `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
-	Time            CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"`
+	Time            CustomTime `json:"time" gorm:"type:timestamp(3); index:idx_time,idx_time_user" swaggertype:"primitive,number"`
 	Hash            string     `json:"-" gorm:"type:varchar(17); uniqueIndex"`
 	Origin          string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
 	OriginId        string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
-	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
+	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp(3)" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
 }
 
 func (h *Heartbeat) Valid() bool {
 	return h.User != nil && h.UserID != "" && h.User.ID == h.UserID && h.Time != CustomTime(time.Time{})
 }
 
+func (h *Heartbeat) Timely(maxAge time.Duration) bool {
+	now := time.Now()
+	return now.Sub(h.Time.T()) <= maxAge && h.Time.T().Sub(now) < 1*time.Hour
+}
+
 func (h *Heartbeat) Augment(languageMappings map[string]string) {
 	maxPrec := -1 // precision / mapping complexity -> more concrete ones shall take precedence
 	for ending, value := range languageMappings {

models/metrics/counter_metric.go

@@ -4,7 +4,7 @@ import "fmt"
 
 type CounterMetric struct {
 	Name   string
-	Value  int
+	Value  int64
 	Desc   string
 	Labels Labels
 }

models/summary.go

@@ -101,7 +101,23 @@ func (s *Summary) MappedItems() map[uint8]*SummaryItems {
 }
 
 func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
-	return s.MappedItems()[summaryType]
+	switch summaryType {
+	case SummaryProject:
+		return &s.Projects
+	case SummaryLanguage:
+		return &s.Languages
+	case SummaryEditor:
+		return &s.Editors
+	case SummaryOS:
+		return &s.OperatingSystems
+	case SummaryMachine:
+		return &s.Machines
+	case SummaryLabel:
+		return &s.Labels
+	case SummaryBranch:
+		return &s.Branches
+	}
+	return nil
 }
 
 func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {

repositories/heartbeat.go

@@ -1,6 +1,7 @@
 package repositories
 
 import (
+	conf "github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
@@ -8,11 +9,12 @@ import (
 )
 
 type HeartbeatRepository struct {
-	db *gorm.DB
+	db     *gorm.DB
+	config *conf.Config
 }
 
 func NewHeartbeatRepository(db *gorm.DB) *HeartbeatRepository {
-	return &HeartbeatRepository{db: db}
+	return &HeartbeatRepository{config: conf.Get(), db: db}
 }
 
 // Use with caution!!
@@ -116,12 +118,19 @@ func (r *HeartbeatRepository) GetLastByUsers() ([]*models.TimeByUser, error) {
 	return result, nil
 }
 
-func (r *HeartbeatRepository) Count() (int64, error) {
-	var count int64
-	if err := r.db.
-		Model(&models.Heartbeat{}).
-		Count(&count).Error; err != nil {
-		return 0, err
+func (r *HeartbeatRepository) Count(approximate bool) (count int64, err error) {
+	if r.config.Db.IsMySQL() && approximate {
+		err = r.db.Table("information_schema.tables").
+			Select("table_rows").
+			Where("table_schema = ?", r.config.Db.Name).
+			Where("table_name = 'heartbeats'").
+			Scan(&count).Error
+	}
+
+	if count == 0 {
+		err = r.db.
+			Model(&models.Heartbeat{}).
+			Count(&count).Error
 	}
 	return count, nil
 }
@@ -145,6 +154,10 @@ func (r *HeartbeatRepository) CountByUsers(users []*models.User) ([]*models.Coun
 		userIds[i] = u.ID
 	}
 
+	if len(userIds) == 0 {
+		return counts, nil
+	}
+
 	if err := r.db.
 		Model(&models.Heartbeat{}).
 		Select("user_id as user, count(id) as count").
@@ -153,6 +166,7 @@ func (r *HeartbeatRepository) CountByUsers(users []*models.User) ([]*models.Coun
 		Find(&counts).Error; err != nil {
 		return counts, err
 	}
+
 	return counts, nil
 }
 
@@ -176,3 +190,12 @@ func (r *HeartbeatRepository) DeleteBefore(t time.Time) error {
 	}
 	return nil
 }
+
+func (r *HeartbeatRepository) DeleteByUser(user *models.User) error {
+	if err := r.db.
+		Where("user_id = ?", user.ID).
+		Delete(models.Heartbeat{}).Error; err != nil {
+		return err
+	}
+	return nil
+}

repositories/metrics.go

@@ -0,0 +1,43 @@
+package repositories
+
+import (
+	"github.com/muety/wakapi/config"
+	"gorm.io/gorm"
+)
+
+type MetricsRepository struct {
+	config *config.Config
+	db     *gorm.DB
+}
+
+const sizeTplMysql = `
+SELECT SUM(data_length + index_length)
+FROM information_schema.tables
+WHERE table_schema = ?
+GROUP BY table_schema`
+
+const sizeTplPostgres = `SELECT pg_database_size('%s');`
+
+const sizeTplSqlite = `
+SELECT page_count * page_size as size
+FROM pragma_page_count(), pragma_page_size();`
+
+func NewMetricsRepository(db *gorm.DB) *MetricsRepository {
+	return &MetricsRepository{config: config.Get(), db: db}
+}
+
+func (srv *MetricsRepository) GetDatabaseSize() (size int64, err error) {
+	cfg := srv.config.Db
+
+	query := srv.db.Raw("SELECT 0")
+	if cfg.IsMySQL() {
+		query = srv.db.Raw(sizeTplMysql, cfg.Name)
+	} else if cfg.IsPostgres() {
+		query = srv.db.Raw(sizeTplPostgres, cfg.Name)
+	} else if cfg.IsSQLite() {
+		query = srv.db.Raw(sizeTplSqlite)
+	}
+
+	err = query.Scan(&size).Error
+	return size, err
+}

repositories/repositories.go

@@ -25,11 +25,12 @@ type IHeartbeatRepository interface {
 	GetLastByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
-	Count() (int64, error)
+	Count(bool) (int64, error)
 	CountByUser(*models.User) (int64, error)
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsRepository interface {

repositories/summary.go

@@ -18,15 +18,15 @@ func (r *SummaryRepository) GetAll() ([]*models.Summary, error) {
 	var summaries []*models.Summary
 	if err := r.db.
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -44,15 +44,15 @@ func (r *SummaryRepository) GetByUserWithin(user *models.User, from, to time.Tim
 		Where("from_time >= ?", from.Local()).
 		Where("to_time <= ?", to.Local()).
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -74,3 +74,32 @@ func (r *SummaryRepository) DeleteByUser(userId string) error {
 	}
 	return nil
 }
+
+// inplace
+func (r *SummaryRepository) populateItems(summaries []*models.Summary) error {
+	summaryMap := map[uint]*models.Summary{}
+	summaryIds := make([]uint, len(summaries))
+	for i, s := range summaries {
+		if s.NumHeartbeats == 0 {
+			continue
+		}
+		summaryMap[s.ID] = s
+		summaryIds[i] = s.ID
+	}
+
+	var items []*models.SummaryItem
+
+	if err := r.db.
+		Model(&models.SummaryItem{}).
+		Where("summary_id in ?", summaryIds).
+		Find(&items).Error; err != nil {
+		return err
+	}
+
+	for _, item := range items {
+		l := summaryMap[item.SummaryID].ItemsByType(item.Type)
+		*l = append(*l, item)
+	}
+
+	return nil
+}

repositories/user.go

@@ -98,10 +98,9 @@ func (r *UserRepository) GetByLoggedInAfter(t time.Time) ([]*models.User, error)
 // Returns a list of user ids, whose last heartbeat is not older than t
 // NOTE: Only ID field will be populated
 func (r *UserRepository) GetByLastActiveAfter(t time.Time) ([]*models.User, error) {
-	subQuery1 := r.db.Model(&models.User{}).
-		Select("users.id as user, max(time) as time").
-		Joins("left join heartbeats on users.id = heartbeats.user_id").
-		Group("user")
+	subQuery1 := r.db.Model(&models.Heartbeat{}).
+		Select("user_id as user, max(time) as time").
+		Group("user_id")
 
 	var userIds []string
 	if err := r.db.

routes/api/diagnostics.go

@@ -46,20 +46,12 @@ func (h *DiagnosticsApiHandler) RegisterRoutes(router *mux.Router) {
 func (h *DiagnosticsApiHandler) Post(w http.ResponseWriter, r *http.Request) {
 	var diagnostics models.Diagnostics
 
-	user := middlewares.GetPrincipal(r)
-	if user == nil {
-		w.WriteHeader(http.StatusUnauthorized)
-		w.Write([]byte(conf.ErrUnauthorized))
-		return
-	}
-
 	if err := json.NewDecoder(r.Body).Decode(&diagnostics); err != nil {
 		w.WriteHeader(http.StatusBadRequest)
 		w.Write([]byte(conf.ErrBadRequest))
 		conf.Log().Request(r).Error("failed to parse diagnostics for user %s - %v", err)
 		return
 	}
-	diagnostics.UserID = user.ID
 
 	if _, err := h.diagnosticsSrvc.Create(&diagnostics); err != nil {
 		w.WriteHeader(http.StatusInternalServerError)

routes/api/heartbeat.go

@@ -86,7 +86,7 @@ func (h *HeartbeatApiHandler) Post(w http.ResponseWriter, r *http.Request) {
 		hb.UserID = user.ID
 		hb.UserAgent = userAgent
 
-		if !hb.Valid() {
+		if !hb.Valid() || !hb.Timely(h.config.App.HeartbeatsMaxAge()) {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write([]byte("invalid heartbeat object"))
 			return

routes/api/metrics.go

@@ -9,6 +9,7 @@ import (
 	"github.com/muety/wakapi/models"
 	v1 "github.com/muety/wakapi/models/compat/wakatime/v1"
 	mm "github.com/muety/wakapi/models/metrics"
+	"github.com/muety/wakapi/repositories"
 	"github.com/muety/wakapi/services"
 	"github.com/muety/wakapi/utils"
 	"net/http"
@@ -39,6 +40,7 @@ const (
 	DescMemAllocTotal = "Total number of bytes allocated for heap"
 	DescMemSysTotal   = "Total number of bytes obtained from the OS"
 	DescGoroutines    = "Total number of running goroutines"
+	DescDatabaseSize  = "Total database size in bytes"
 )
 
 type MetricsHandler struct {
@@ -47,14 +49,16 @@ type MetricsHandler struct {
 	summarySrvc   services.ISummaryService
 	heartbeatSrvc services.IHeartbeatService
 	keyValueSrvc  services.IKeyValueService
+	metricsRepo   *repositories.MetricsRepository
 }
 
-func NewMetricsHandler(userService services.IUserService, summaryService services.ISummaryService, heartbeatService services.IHeartbeatService, keyValueService services.IKeyValueService) *MetricsHandler {
+func NewMetricsHandler(userService services.IUserService, summaryService services.ISummaryService, heartbeatService services.IHeartbeatService, keyValueService services.IKeyValueService, metricsRepo *repositories.MetricsRepository) *MetricsHandler {
 	return &MetricsHandler{
 		userSrvc:      userService,
 		summarySrvc:   summaryService,
 		heartbeatSrvc: heartbeatService,
 		keyValueSrvc:  keyValueService,
+		metricsRepo:   metricsRepo,
 		config:        conf.Get(),
 	}
 }
@@ -141,21 +145,21 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_cumulative_seconds_total",
 		Desc:   DescAllTime,
-		Value:  int(v1.NewAllTimeFrom(summaryAllTime).Data.TotalSeconds),
+		Value:  int64(v1.NewAllTimeFrom(summaryAllTime).Data.TotalSeconds),
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_seconds_total",
 		Desc:   DescTotal,
-		Value:  int(summaryToday.TotalTime().Seconds()),
+		Value:  int64(summaryToday.TotalTime().Seconds()),
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_heartbeats_total",
 		Desc:   DescHeartbeats,
-		Value:  int(heartbeatCount),
+		Value:  int64(heartbeatCount),
 		Labels: []mm.Label{},
 	})
 
@@ -163,7 +167,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_project_seconds_total",
 			Desc:   DescProjects,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryProject, p.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryProject, p.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: p.Key}},
 		})
 	}
@@ -172,7 +176,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_language_seconds_total",
 			Desc:   DescLanguages,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryLanguage, l.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryLanguage, l.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: l.Key}},
 		})
 	}
@@ -181,7 +185,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_editor_seconds_total",
 			Desc:   DescEditors,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryEditor, e.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryEditor, e.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: e.Key}},
 		})
 	}
@@ -190,7 +194,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_operating_system_seconds_total",
 			Desc:   DescOperatingSystems,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryOS, o.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryOS, o.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: o.Key}},
 		})
 	}
@@ -199,7 +203,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_machine_seconds_total",
 			Desc:   DescMachines,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryMachine, m.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryMachine, m.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: m.Key}},
 		})
 	}
@@ -208,7 +212,7 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_label_seconds_total",
 			Desc:   DescLabels,
-			Value:  int(summaryToday.TotalTimeByKey(models.SummaryLabel, m.Key).Seconds()),
+			Value:  int64(summaryToday.TotalTimeByKey(models.SummaryLabel, m.Key).Seconds()),
 			Labels: []mm.Label{{Key: "name", Value: m.Key}},
 		})
 	}
@@ -220,21 +224,34 @@ func (h *MetricsHandler) getUserMetrics(user *models.User) (*mm.Metrics, error)
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_goroutines_total",
 		Desc:   DescGoroutines,
-		Value:  runtime.NumGoroutine(),
+		Value:  int64(runtime.NumGoroutine()),
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_mem_alloc_total",
 		Desc:   DescMemAllocTotal,
-		Value:  int(memStats.Alloc),
+		Value:  int64(memStats.Alloc),
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_mem_sys_total",
 		Desc:   DescMemSysTotal,
-		Value:  int(memStats.Sys),
+		Value:  int64(memStats.Sys),
+		Labels: []mm.Label{},
+	})
+
+	// Database metrics
+	dbSize, err := h.metricsRepo.GetDatabaseSize()
+	if err != nil {
+		logbuch.Warn("failed to get database size (%v)", err)
+	}
+
+	metrics = append(metrics, &mm.CounterMetric{
+		Name:   MetricsPrefix + "_db_total_bytes",
+		Desc:   DescDatabaseSize,
+		Value:  dbSize,
 		Labels: []mm.Label{},
 	})
 
@@ -256,7 +273,7 @@ func (h *MetricsHandler) getAdminMetrics(user *models.User) (*mm.Metrics, error)
 	}
 
 	totalUsers, _ := h.userSrvc.Count()
-	totalHeartbeats, _ := h.heartbeatSrvc.Count()
+	totalHeartbeats, _ := h.heartbeatSrvc.Count(true)
 
 	activeUsers, err := h.userSrvc.GetActive(false)
 	if err != nil {
@@ -267,28 +284,28 @@ func (h *MetricsHandler) getAdminMetrics(user *models.User) (*mm.Metrics, error)
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_admin_seconds_total",
 		Desc:   DescAdminTotalTime,
-		Value:  totalSeconds,
+		Value:  int64(totalSeconds),
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_admin_heartbeats_total",
 		Desc:   DescAdminTotalHeartbeats,
-		Value:  int(totalHeartbeats),
+		Value:  totalHeartbeats,
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_admin_users_total",
 		Desc:   DescAdminTotalUsers,
-		Value:  int(totalUsers),
+		Value:  totalUsers,
 		Labels: []mm.Label{},
 	})
 
 	metrics = append(metrics, &mm.CounterMetric{
 		Name:   MetricsPrefix + "_admin_users_active_total",
 		Desc:   DescAdminActiveUsers,
-		Value:  len(activeUsers),
+		Value:  int64(len(activeUsers)),
 		Labels: []mm.Label{},
 	})
 
@@ -304,7 +321,7 @@ func (h *MetricsHandler) getAdminMetrics(user *models.User) (*mm.Metrics, error)
 		metrics = append(metrics, &mm.CounterMetric{
 			Name:   MetricsPrefix + "_admin_user_heartbeats_total",
 			Desc:   DescAdminUserHeartbeats,
-			Value:  int(uc.Count),
+			Value:  uc.Count,
 			Labels: []mm.Label{{Key: "user", Value: uc.User}},
 		})
 	}

routes/settings.go

@@ -151,6 +151,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
 		return h.actionImportWakatime
 	case "regenerate_summaries":
 		return h.actionRegenerateSummaries
+	case "clear_data":
+		return h.actionClearData
 	case "delete_account":
 		return h.actionDeleteUser
 	}
@@ -553,6 +555,29 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
 	return http.StatusAccepted, "summaries are being regenerated - this may take a up to a couple of minutes, please come back later", ""
 }
 
+func (h *SettingsHandler) actionClearData(w http.ResponseWriter, r *http.Request) (int, string, string) {
+	if h.config.IsDev() {
+		loadTemplates()
+	}
+
+	user := middlewares.GetPrincipal(r)
+	logbuch.Info("user '%s' requested to delete all data", user.ID)
+
+	go func(user *models.User) {
+		logbuch.Info("deleting summaries for user '%s'", user.ID)
+		if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+			logbuch.Error("failed to clear summaries: %v", err)
+		}
+
+		logbuch.Info("deleting heartbeats for user '%s'", user.ID)
+		if err := h.heartbeatSrvc.DeleteByUser(user); err != nil {
+			logbuch.Error("failed to clear heartbeats: %v", err)
+		}
+	}(user)
+
+	return http.StatusAccepted, "deletion in progress, this may take a couple of seconds", ""
+}
+
 func (h *SettingsHandler) actionDeleteUser(w http.ResponseWriter, r *http.Request) (int, string, string) {
 	if h.config.IsDev() {
 		loadTemplates()

scripts/aggregate_durations.sql

@@ -0,0 +1,9 @@
+SELECT project, language, editor, operating_system, machine, branch, SUM(GREATEST(1, diff)) as 'sum'
+FROM (
+         SELECT project, language, editor, operating_system, machine, branch, TIME_TO_SEC(LEAST(TIMEDIFF(time, LAG(time) over w), '00:02:00')) as 'diff'
+         FROM heartbeats
+         WHERE user_id = 'n1try'
+             WINDOW w AS (ORDER BY time)
+     ) s2
+WHERE diff IS NOT NULL
+GROUP BY project, language, editor, operating_system, machine, branch;

scripts/clean_duplicates.sql

@@ -0,0 +1,12 @@
+DELETE t1
+FROM heartbeats t1
+         INNER JOIN heartbeats t2
+WHERE t1.id < t2.id
+  AND t1.time = t2.time
+  AND t1.entity = t2.entity
+  AND t1.is_write = t2.is_write
+  AND t1.branch = t2.branch
+  AND t1.editor = t2.editor
+  AND t1.machine = t2.machine
+  AND t1.operating_system = t2.operating_system
+  AND t1.user_id = t2.user_id;

scripts/count_duplicates_by_user.sql

@@ -0,0 +1,10 @@
+SELECT s2.user_id, sum(c) as count, total, (sum(c) / total) as ratio
+FROM (
+         SELECT time, user_id, entity, is_write, branch, editor, machine, operating_system, COUNT(time) as c
+         FROM heartbeats
+         GROUP BY time, user_id, entity, is_write, branch, editor, machine, operating_system
+         HAVING COUNT(time) > 1
+     ) s2
+         LEFT JOIN (SELECT user_id, count(id) AS total FROM heartbeats GROUP BY user_id) s3 ON s2.user_id = s3.user_id
+GROUP BY user_id
+ORDER BY count DESC;

services/duration.go

@@ -36,6 +36,8 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
 	}
 
 	// Aggregation
+	// the below logic is approximately equivalent to the SQL query at scripts/aggregate_durations.sql,
+	// but unfortunately we cannot use it, as it features mysql-specific functions (lag(), timediff(), ...)
 	var count int
 	var latest *models.Duration
 
@@ -80,12 +82,20 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
 
 	for _, list := range mapping {
 		for _, d := range list {
+			// will only happen if two heartbeats with different hashes (e.g. different project) have the same timestamp
+			// that, in turn, will most likely only happen for mysql, where `time` column's precision was set to second for a while
+			// assume that two non-identical heartbeats with identical time are sub-second apart from each other, so round up to expectancy value
+			// also see https://github.com/muety/wakapi/issues/340
 			if d.Duration == 0 {
-				d.Duration = HeartbeatDiffThreshold
+				d.Duration = 500 * time.Millisecond
 			}
 			durations = append(durations, d)
 		}
 	}
 
+	if len(heartbeats) == 1 && len(durations) == 1 {
+		durations[0].Duration = HeartbeatDiffThreshold
+	}
+
 	return durations.Sorted(), nil
 }

services/duration_test.go

@@ -65,6 +65,17 @@ func (suite *DurationServiceTestSuite) SetupSuite() {
 			Machine:         TestMachine1,
 			Time:            models.CustomTime(suite.TestStartTime.Add(30 * time.Second)), // 0:30
 		},
+		// duplicate of previous one
+		{
+			ID:              rand.Uint64(),
+			UserID:          TestUserId,
+			Project:         TestProject1,
+			Language:        TestLanguageGo,
+			Editor:          TestEditorGoland,
+			OperatingSystem: TestOsLinux,
+			Machine:         TestMachine1,
+			Time:            models.CustomTime(suite.TestStartTime.Add(30 * time.Second)), // 0:30
+		},
 		{
 			ID:              rand.Uint64(),
 			UserID:          TestUserId,
@@ -160,7 +171,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get() {
 	assert.Equal(suite.T(), TestEditorGoland, durations[0].Editor)
 	assert.Equal(suite.T(), TestEditorGoland, durations[1].Editor)
 	assert.Equal(suite.T(), TestEditorVscode, durations[2].Editor)
-	assert.Equal(suite.T(), 2, durations[0].NumHeartbeats)
+	assert.Equal(suite.T(), 3, durations[0].NumHeartbeats)
 	assert.Equal(suite.T(), 1, durations[1].NumHeartbeats)
 	assert.Equal(suite.T(), 3, durations[2].NumHeartbeats)
 }

services/heartbeat.go

@@ -73,12 +73,12 @@ func (srv *HeartbeatService) InsertBatch(heartbeats []*models.Heartbeat) error {
 	return err
 }
 
-func (srv *HeartbeatService) Count() (int64, error) {
+func (srv *HeartbeatService) Count(approximate bool) (int64, error) {
 	result, ok := srv.cache.Get(srv.countTotalCacheKey())
 	if ok {
 		return result.(int64), nil
 	}
-	count, err := srv.repository.Count()
+	count, err := srv.repository.Count(approximate)
 	if err == nil {
 		srv.cache.Set(srv.countTotalCacheKey(), count, srv.countCacheTtl())
 	}
@@ -179,9 +179,15 @@ func (srv *HeartbeatService) GetEntitySetByUser(entityType uint8, user *models.U
 }
 
 func (srv *HeartbeatService) DeleteBefore(t time.Time) error {
+	go srv.cache.Flush()
 	return srv.repository.DeleteBefore(t)
 }
 
+func (srv *HeartbeatService) DeleteByUser(user *models.User) error {
+	go srv.cache.Flush()
+	return srv.repository.DeleteByUser(user)
+}
+
 func (srv *HeartbeatService) augmented(heartbeats []*models.Heartbeat, userId string) ([]*models.Heartbeat, error) {
 	languageMapping, err := srv.languageMappingSrvc.ResolveByUser(userId)
 	if err != nil {

services/services.go

@@ -29,7 +29,7 @@ type IAliasService interface {
 type IHeartbeatService interface {
 	Insert(*models.Heartbeat) error
 	InsertBatch([]*models.Heartbeat) error
-	Count() (int64, error)
+	Count(bool) (int64, error)
 	CountByUser(*models.User) (int64, error)
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
@@ -39,6 +39,7 @@ type IHeartbeatService interface {
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsService interface {

static/assets/js/components/settings.js

@@ -21,6 +21,11 @@ PetiteVue.createApp({
             document.querySelector('#form-import-wakatime').submit()
         }
     },
+    confirmClearData() {
+        if (confirm('Are you sure? This can not be undone!')) {
+            document.querySelector('#form-clear-data').submit()
+        }
+    },
     confirmDeleteAccount() {
         if (confirm('Are you sure? This can not be undone!')) {
             document.querySelector('#form-delete-user').submit()

version.txt

@@ -1 +1 @@
-2.2.6
+2.3.1

views/settings.tpl.html

@@ -609,7 +609,7 @@
                             Regenerate all pre-computed summaries from raw heartbeat data. This may be useful if, for some reason, summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively). This may take some time. Be careful and only run this action if you know, what your are doing, as data loss might occur.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmRegenerate">Clear and regenerate</button>
                     </div>
                 </form>
@@ -623,11 +623,25 @@
                             Please note that resetting your API key requires you to update your .wakatime.cfg files on all of your computers to make the WakaTime client send heartbeats again.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="submit" class="btn-danger ml-1">Reset API key</button>
                     </div>
                 </form>
 
+                <form action="" method="post" class="flex mb-8" id="form-clear-data">
+                    <input type="hidden" name="action" value="clear_data">
+
+                    <div class="w-1/2 mr-4 inline-block">
+                        <span class="font-semibold text-gray-300">Clear Data</span>
+                        <span class="block text-sm text-gray-600">
+                            Clear all your time tracking data from Wakapi. This cannot be undone. Be careful!
+                        </span>
+                    </div>
+                    <div class="w-1/2 ml-4 flex items-center">
+                        <button type="button" class="btn-danger ml-1" @click.stop="confirmClearData">Clear data</button>
+                    </div>
+                </form>
+
                 <form action="" method="post" class="flex mb-8" id="form-delete-user">
                     <input type="hidden" name="action" value="delete_account">
 
@@ -637,7 +651,7 @@
                             Deleting your account will cause all data, including all your heartbeats, to be erased from the server immediately. This action is irreversible. Be careful!
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmDeleteAccount">Delete account</button>
                     </div>
                 </form>