feat: ability to clear all user data (resolve #339)

chore: add migration note

Dockerfile

@@ -56,4 +56,6 @@ COPY --from=build-env /app .
 
 VOLUME /data
 
+EXPOSE 3000
+
 ENTRYPOINT /app/entrypoint.sh

README.md

@@ -128,6 +128,11 @@ You can specify configuration options either via a config file (default: `config
 | YAML Key / Env. Variable                                                     | Default                                          | Description                                                                                                                                                              |
 |------------------------------------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `env` /<br>`ENVIRONMENT`                                                     | `dev`                                            | Whether to use development- or production settings                                                                                                                       |
+| `app.aggregation_time`<br>`WAKAPI_AGGREGATION_TIME`                          | `02:15`                                          | Time of day at which to periodically run summary generation for all users                                                                                                |
+| `app.report_time_weekly`<br>`WAKAPI_REPORT_TIME_WEEKLY`                      | `fri,18:00`                                      | Week day and time at which to send e-mail reports                                                                                                                        |
+| `app.import_batch_size`<br>`WAKAPI_IMPORT_BATCH_SIZE`                        | `50`                                             | Size of batches of heartbeats to insert to the database during importing from external services                                                                          |
+| `app.inactive_days`<br>`WAKAPI_INACTIVE_DAYS`                                | `7`                                              | Number of days after which to consider a user inactive (only for metrics)                                                                                                |
+| `app.heartbeat_max_age`<br>`WAKAPI_HEARTBEAT_MAX_AGE`                        | `4320h`                                          | Maximum acceptable age of a heartbeat (see [`ParseDuration`](https://pkg.go.dev/time#ParseDuration))                                                                     |
 | `app.custom_languages`                                                       | -                                                | Map from file endings to language names                                                                                                                                  |
 | `app.avatar_url_template`                                                    | (see [`config.default.yml`](config.default.yml)) | URL template for external user avatar images (e.g. from [Dicebear](https://dicebear.com) or [Gravatar](https://gravatar.com))                                            |
 | `server.port` /<br> `WAKAPI_PORT`                                            | `3000`                                           | Port to listen on                                                                                                                                                        |
@@ -154,10 +159,16 @@ You can specify configuration options either via a config file (default: `config
 | `db.ssl` /<br> `WAKAPI_DB_SSL`                                               | `false`                                          | Whether to use TLS encryption for database connection (Postgres and CockroachDB only)                                                                                    |
 | `db.automgirate_fail_silently` /<br> `WAKAPI_DB_AUTOMIGRATE_FAIL_SILENTLY`   | `false`                                          | Whether to ignore schema auto-migration failures when starting up                                                                                                        |
 | `mail.enabled` /<br> `WAKAPI_MAIL_ENABLED`                                   | `true`                                           | Whether to allow Wakapi to send e-mail (e.g. for password resets)                                                                                                        |
-| `mail.sender` /<br> `WAKAPI_MAIL_SENDER`                                     | `noreply@wakapi.dev`                             | Default sender address for outgoing mails (ignored for MailWhale)                                                                                                        |
+| `mail.sender` /<br> `WAKAPI_MAIL_SENDER`                                     | `Wakapi <noreply@wakapi.dev>`                    | Default sender address for outgoing mails (ignored for MailWhale)                                                                                                        |
 | `mail.provider` /<br> `WAKAPI_MAIL_PROVIDER`                                 | `smtp`                                           | Implementation to use for sending mails (one of [`smtp`, `mailwhale`])                                                                                                   |
-| `mail.smtp.*` /<br> `WAKAPI_MAIL_SMTP_*`                                     | `-`                                              | Various options to configure SMTP. See [default config](config.default.yml) for details                                                                                  |
-| `mail.mailwhale.*` /<br> `WAKAPI_MAIL_MAILWHALE_*`                           | `-`                                              | Various options to configure [MailWhale](https://mailwhale.dev) sending service. See [default config](config.default.yml) for details                                    |
+| `mail.smtp.host` /<br> `WAKAPI_MAIL_SMTP_HOST`                               | -                                                | SMTP server address for sending mail (if using `smtp` mail provider)                                                                                                     |
+| `mail.smtp.port` /<br> `WAKAPI_MAIL_SMTP_PORT`                               | -                                                | SMTP server port (usually 465)                                                                                                                                           |
+| `mail.smtp.username` /<br> `WAKAPI_MAIL_SMTP_USER`                           | -                                                | SMTP server authentication username                                                                                                                                      |
+| `mail.smtp.password` /<br> `WAKAPI_MAIL_SMTP_PASS`                           | -                                                | SMTP server authentication password                                                                                                                                      |
+| `mail.smtp.tls` /<br> `WAKAPI_MAIL_SMTP_TLS`                                 | `false`                                          | Whether the SMTP server requires TLS encryption (`false` for STARTTLS or no encryption)                                                                                  |
+| `mail.mailwhale.url` /<br> `WAKAPI_MAIL_MAILWHALE_URL`                       | -                                                | URL of [MailWhale](https://mailwhale.dev) instance (e.g. `https://mailwhale.dev`) (if using `mailwhale` mail provider`)                                                  |
+| `mail.mailwhale.client_id` /<br> `WAKAPI_MAIL_MAILWHALE_CLIENT_ID`           | -                                                | MailWhale API client ID                                                                                                                                                  |
+| `mail.mailwhale.client_secret` /<br> `WAKAPI_MAIL_MAILWHALE_CLIENT_SECRET`   | -                                                | MailWhale API client secret                                                                                                                                              |
 | `sentry.dsn` /<br> `WAKAPI_SENTRY_DSN`                                       | –                                                | DSN for to integrate [Sentry](https://sentry.io) for error logging and tracing (leave empty to disable)                                                                  |
 | `sentry.enable_tracing` /<br> `WAKAPI_SENTRY_TRACING`                        | `false`                                          | Whether to enable Sentry request tracing                                                                                                                                 |
 | `sentry.sample_rate` /<br> `WAKAPI_SENTRY_SAMPLE_RATE`                       | `0.75`                                           | Probability of tracing a request in Sentry                                                                                                                               |

config.default.yml

@@ -16,6 +16,7 @@ app:
   report_time_weekly: 'fri,18:00'     # time at which to fan out weekly reports (format: '<weekday)>,<daytime>')
   inactive_days: 7                    # time of previous days within a user must have logged in to be considered active
   import_batch_size: 50               # maximum number of heartbeats to insert into the database within one transaction
+  heartbeat_max_age: '4320h'          # maximum acceptable age of a heartbeat (see https://pkg.go.dev/time#ParseDuration)
   custom_languages:
     vue: Vue
     jsx: JSX

config/config.go

@@ -68,6 +68,7 @@ type appConfig struct {
 	ImportBackoffMin  int                          `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"`
 	ImportBatchSize   int                          `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"`
 	InactiveDays      int                          `yaml:"inactive_days" default:"7" env:"WAKAPI_INACTIVE_DAYS"`
+	HeartbeatMaxAge   string                       `yaml:"heartbeat_max_age" default:"4320h" env:"WAKAPI_HEARTBEAT_MAX_AGE"`
 	CountCacheTTLMin  int                          `yaml:"count_cache_ttl_min" default:"30" env:"WAKAPI_COUNT_CACHE_TTL_MIN"`
 	AvatarURLTemplate string                       `yaml:"avatar_url_template" default:"api/avatar/{username_hash}.svg"`
 	CustomLanguages   map[string]string            `yaml:"custom_languages"`
@@ -242,6 +243,11 @@ func (c *appConfig) GetWeeklyReportTime() string {
 	return strings.Split(c.ReportTimeWeekly, ",")[1]
 }
 
+func (c *appConfig) HeartbeatsMaxAge() time.Duration {
+	d, _ := time.ParseDuration(c.HeartbeatMaxAge)
+	return d
+}
+
 func (c *dbConfig) IsSQLite() bool {
 	return c.Dialect == "sqlite3"
 }
@@ -400,6 +406,9 @@ func Load(version string) *Config {
 	if _, err := time.Parse("15:04", config.App.AggregationTime); err != nil {
 		logbuch.Fatal("invalid interval set for aggregation_time")
 	}
+	if _, err := time.ParseDuration(config.App.HeartbeatMaxAge); err != nil {
+		logbuch.Fatal("invalid duration set for heartbeat_max_age")
+	}
 
 	Set(config)
 	return Get()

migrations/20220317_align_num_heartbeats.go

@@ -0,0 +1,56 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220317-align_num_heartbeats"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			logbuch.Info("this may take a while!")
+
+			// find all summaries whose num_heartbeats is zero even though they have items
+			var faultyIds []uint
+
+			if err := db.Model(&models.Summary{}).
+				Distinct("summaries.id").
+				Joins("INNER JOIN summary_items ON summaries.num_heartbeats = 0 AND summaries.id = summary_items.summary_id").
+				Scan(&faultyIds).Error; err != nil {
+				return err
+			}
+
+			// update their heartbeats counter
+			result := db.
+				Table("summaries AS s1").
+				Where("s1.id IN ?", faultyIds).
+				Update(
+					"num_heartbeats",
+					db.
+						Model(&models.Heartbeat{}).
+						Select("COUNT(*)").
+						Where("user_id = ?", gorm.Expr("s1.user_id")).
+						Where("time BETWEEN ? AND ?", gorm.Expr("s1.from_time"), gorm.Expr("s1.to_time")),
+				)
+
+			if err := result.Error; err != nil {
+				return err
+			}
+
+			logbuch.Info("corrected heartbeats counter of %d summaries", result.RowsAffected)
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

mocks/heartbeat_service.go

@@ -40,6 +40,11 @@ func (m *HeartbeatServiceMock) GetAllWithin(time time.Time, time2 time.Time, use
 	return args.Get(0).([]*models.Heartbeat), args.Error(1)
 }
 
+func (m *HeartbeatServiceMock) GetAllWithinByFilters(time time.Time, time2 time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	args := m.Called(time, time2, user, filters)
+	return args.Get(0).([]*models.Heartbeat), args.Error(1)
+}
+
 func (m *HeartbeatServiceMock) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	args := m.Called()
 	return args.Get(0).([]*models.TimeByUser), args.Error(1)
@@ -64,3 +69,8 @@ func (m *HeartbeatServiceMock) DeleteBefore(time time.Time) error {
 	args := m.Called(time)
 	return args.Error(0)
 }
+
+func (m *HeartbeatServiceMock) DeleteByUser(u *models.User) error {
+	args := m.Called(u)
+	return args.Error(0)
+}

models/filters.go

@@ -92,7 +92,7 @@ func (f *Filters) OneOrEmpty() FilterElement {
 	if ok, t, of := f.One(); ok {
 		return FilterElement{entity: t, filter: of}
 	}
-	return FilterElement{}
+	return FilterElement{entity: SummaryUnknown, filter: []string{}}
 }
 
 func (f *Filters) IsEmpty() bool {
@@ -100,6 +100,49 @@ func (f *Filters) IsEmpty() bool {
 	return !nonEmpty
 }
 
+func (f *Filters) Count() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		count += f.CountByEntity(i)
+	}
+	return count
+}
+
+func (f *Filters) CountByEntity(entity uint8) int {
+	return len(*f.ResolveEntity(entity))
+}
+
+func (f *Filters) EntityCount() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		if c := f.CountByEntity(i); c > 0 {
+			count++
+		}
+	}
+	return count
+}
+
+func (f *Filters) ResolveEntity(entityId uint8) *OrFilter {
+	switch entityId {
+	case SummaryProject:
+		return &f.Project
+	case SummaryLanguage:
+		return &f.Language
+	case SummaryEditor:
+		return &f.Editor
+	case SummaryOS:
+		return &f.OS
+	case SummaryMachine:
+		return &f.Machine
+	case SummaryLabel:
+		return &f.Label
+	case SummaryBranch:
+		return &f.Branch
+	default:
+		return &OrFilter{}
+	}
+}
+
 func (f *Filters) Hash() string {
 	hash, err := hashstructure.Hash(f, hashstructure.FormatV2, nil)
 	if err != nil {

models/heartbeat.go

@@ -12,21 +12,21 @@ type Heartbeat struct {
 	ID              uint64     `gorm:"primary_key" hash:"ignore"`
 	User            *User      `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" hash:"ignore"`
 	UserID          string     `json:"-" gorm:"not null; index:idx_time_user"`
-	Entity          string     `json:"entity" gorm:"not null; index:idx_entity"`
+	Entity          string     `json:"entity" gorm:"not null"`
 	Type            string     `json:"type"`
 	Category        string     `json:"category"`
-	Project         string     `json:"project"`
-	Branch          string     `json:"branch"`
+	Project         string     `json:"project" gorm:"index:idx_project"`
+	Branch          string     `json:"branch" gorm:"index:idx_branch"`
 	Language        string     `json:"language" gorm:"index:idx_language"`
 	IsWrite         bool       `json:"is_write"`
-	Editor          string     `json:"editor" hash:"ignore"`           // ignored because editor might be parsed differently by wakatime
-	OperatingSystem string     `json:"operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
-	Machine         string     `json:"machine" hash:"ignore"`          // ignored because wakatime api doesn't return machines currently
-	UserAgent       string     `json:"user_agent" hash:"ignore"`
+	Editor          string     `json:"editor" gorm:"index:idx_editor" hash:"ignore"`                     // ignored because editor might be parsed differently by wakatime
+	OperatingSystem string     `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
+	Machine         string     `json:"machine" gorm:"index:idx_machine" hash:"ignore"`                   // ignored because wakatime api doesn't return machines currently
+	UserAgent       string     `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
 	Time            CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"`
 	Hash            string     `json:"-" gorm:"type:varchar(17); uniqueIndex"`
-	Origin          string     `json:"-" hash:"ignore"`
-	OriginId        string     `json:"-" hash:"ignore"`
+	Origin          string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
+	OriginId        string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
 	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
 }
 
@@ -34,6 +34,11 @@ func (h *Heartbeat) Valid() bool {
 	return h.User != nil && h.UserID != "" && h.User.ID == h.UserID && h.Time != CustomTime(time.Time{})
 }
 
+func (h *Heartbeat) Timely(maxAge time.Duration) bool {
+	now := time.Now()
+	return now.Sub(h.Time.T()) <= maxAge && h.Time.T().Before(now)
+}
+
 func (h *Heartbeat) Augment(languageMappings map[string]string) {
 	maxPrec := -1 // precision / mapping complexity -> more concrete ones shall take precedence
 	for ending, value := range languageMappings {
@@ -99,3 +104,15 @@ func (h *Heartbeat) Hashed() *Heartbeat {
 	h.Hash = fmt.Sprintf("%x", hash) // "uint64 values with high bit set are not supported"
 	return h
 }
+
+func GetEntityColumn(t uint8) string {
+	return []string{
+		"project",
+		"language",
+		"editor",
+		"operating_system",
+		"machine",
+		"label",
+		"branch",
+	}[t]
+}

models/summary.go

@@ -8,6 +8,7 @@ import (
 
 const (
 	NSummaryTypes   uint8 = 99
+	SummaryUnknown  uint8 = 98
 	SummaryProject  uint8 = 0
 	SummaryLanguage uint8 = 1
 	SummaryEditor   uint8 = 2
@@ -100,7 +101,37 @@ func (s *Summary) MappedItems() map[uint8]*SummaryItems {
 }
 
 func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
-	return s.MappedItems()[summaryType]
+	switch summaryType {
+	case SummaryProject:
+		return &s.Projects
+	case SummaryLanguage:
+		return &s.Languages
+	case SummaryEditor:
+		return &s.Editors
+	case SummaryOS:
+		return &s.OperatingSystems
+	case SummaryMachine:
+		return &s.Machines
+	case SummaryLabel:
+		return &s.Labels
+	case SummaryBranch:
+		return &s.Branches
+	}
+	return nil
+}
+
+func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {
+	if len(types) == 0 {
+		return s
+	}
+
+	for _, t := range SummaryTypes() {
+		if keep, ok := types[t]; !keep || !ok {
+			*s.ItemsByType(t) = []*SummaryItem{}
+		}
+	}
+
+	return s
 }
 
 /* Augments the summary in a way that at least one item is present for every type.

models/summary_test.go

@@ -168,6 +168,66 @@ func TestSummary_WithResolvedAliases(t *testing.T) {
 	assert.Empty(t, sut.Machines)
 }
 
+func TestSummary_KeepOnly(t *testing.T) {
+	newSummary := func() *Summary {
+		return &Summary{
+			Projects: []*SummaryItem{
+				{
+					Type: SummaryProject,
+					Key:  "wakapi",
+					// hack to work around the issue that the total time of a summary item is mistakenly represented in seconds
+					Total: 10 * time.Minute / time.Second,
+				},
+				{
+					Type:  SummaryProject,
+					Key:   "anchr",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Languages: []*SummaryItem{
+				{
+					Type:  SummaryLanguage,
+					Key:   "Go",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Editors: []*SummaryItem{
+				{
+					Type:  SummaryEditor,
+					Key:   "VSCode",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+		}
+	}
+
+	var sut *Summary
+
+	sut = newSummary().KeepOnly(map[uint8]bool{}) // keep all
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.Zero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.Zero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryEditor: true, SummaryLanguage: true})
+	assert.Zero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 10*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	sut.FillMissing()
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+}
+
 func TestSummaryItems_Sorted(t *testing.T) {
 	testDuration1, testDuration2, testDuration3 := 10*time.Minute, 5*time.Minute, 20*time.Minute
 

repositories/heartbeat.go

@@ -1,7 +1,6 @@
 package repositories
 
 import (
-	"errors"
 	"github.com/muety/wakapi/models"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
@@ -77,6 +76,26 @@ func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User
 	return heartbeats, nil
 }
 
+func (r *HeartbeatRepository) GetAllWithinByFilters(from, to time.Time, user *models.User, filterMap map[string][]string) ([]*models.Heartbeat, error) {
+	// https://stackoverflow.com/a/20765152/3112139
+	var heartbeats []*models.Heartbeat
+
+	q := r.db.
+		Where(&models.Heartbeat{UserID: user.ID}).
+		Where("time >= ?", from.Local()).
+		Where("time < ?", to.Local()).
+		Order("time asc")
+
+	for col, vals := range filterMap {
+		q = q.Where(col+" in ?", vals)
+	}
+
+	if err := q.Find(&heartbeats).Error; err != nil {
+		return nil, err
+	}
+	return heartbeats, nil
+}
+
 func (r *HeartbeatRepository) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	var result []*models.TimeByUser
 	r.db.Model(&models.User{}).
@@ -138,16 +157,10 @@ func (r *HeartbeatRepository) CountByUsers(users []*models.User) ([]*models.Coun
 }
 
 func (r HeartbeatRepository) GetEntitySetByUser(entityType uint8, user *models.User) ([]string, error) {
-	columns := []string{"project", "language", "editor", "operating_system", "machine"}
-	if int(entityType) >= len(columns) {
-		// invalid entity type
-		return nil, errors.New("invalid entity type")
-	}
-
 	var results []string
 	if err := r.db.
 		Model(&models.Heartbeat{}).
-		Distinct(columns[entityType]).
+		Distinct(models.GetEntityColumn(entityType)).
 		Where(&models.Heartbeat{UserID: user.ID}).
 		Find(&results).Error; err != nil {
 		return nil, err
@@ -163,3 +176,12 @@ func (r *HeartbeatRepository) DeleteBefore(t time.Time) error {
 	}
 	return nil
 }
+
+func (r *HeartbeatRepository) DeleteByUser(user *models.User) error {
+	if err := r.db.
+		Where("user_id = ?", user.ID).
+		Delete(models.Heartbeat{}).Error; err != nil {
+		return err
+	}
+	return nil
+}

repositories/repositories.go

@@ -20,6 +20,7 @@ type IHeartbeatRepository interface {
 	InsertBatch([]*models.Heartbeat) error
 	GetAll() ([]*models.Heartbeat, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, map[string][]string) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLastByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)
@@ -29,6 +30,7 @@ type IHeartbeatRepository interface {
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsRepository interface {

repositories/summary.go

@@ -18,15 +18,15 @@ func (r *SummaryRepository) GetAll() ([]*models.Summary, error) {
 	var summaries []*models.Summary
 	if err := r.db.
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -44,15 +44,15 @@ func (r *SummaryRepository) GetByUserWithin(user *models.User, from, to time.Tim
 		Where("from_time >= ?", from.Local()).
 		Where("to_time <= ?", to.Local()).
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -74,3 +74,32 @@ func (r *SummaryRepository) DeleteByUser(userId string) error {
 	}
 	return nil
 }
+
+// inplace
+func (r *SummaryRepository) populateItems(summaries []*models.Summary) error {
+	summaryMap := map[uint]*models.Summary{}
+	summaryIds := make([]uint, len(summaries))
+	for i, s := range summaries {
+		if s.NumHeartbeats == 0 {
+			continue
+		}
+		summaryMap[s.ID] = s
+		summaryIds[i] = s.ID
+	}
+
+	var items []*models.SummaryItem
+
+	if err := r.db.
+		Model(&models.SummaryItem{}).
+		Where("summary_id in ?", summaryIds).
+		Find(&items).Error; err != nil {
+		return err
+	}
+
+	for _, item := range items {
+		l := summaryMap[item.SummaryID].ItemsByType(item.Type)
+		*l = append(*l, item)
+	}
+
+	return nil
+}

routes/api/heartbeat.go

@@ -86,7 +86,7 @@ func (h *HeartbeatApiHandler) Post(w http.ResponseWriter, r *http.Request) {
 		hb.UserID = user.ID
 		hb.UserAgent = userAgent
 
-		if !hb.Valid() {
+		if !hb.Valid() || !hb.Timely(h.config.App.HeartbeatsMaxAge()) {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write([]byte("invalid heartbeat object"))
 			return

routes/settings.go

@@ -151,6 +151,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
 		return h.actionImportWakatime
 	case "regenerate_summaries":
 		return h.actionRegenerateSummaries
+	case "clear_data":
+		return h.actionClearData
 	case "delete_account":
 		return h.actionDeleteUser
 	}
@@ -553,6 +555,29 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
 	return http.StatusAccepted, "summaries are being regenerated - this may take a up to a couple of minutes, please come back later", ""
 }
 
+func (h *SettingsHandler) actionClearData(w http.ResponseWriter, r *http.Request) (int, string, string) {
+	if h.config.IsDev() {
+		loadTemplates()
+	}
+
+	user := middlewares.GetPrincipal(r)
+	logbuch.Info("user '%s' requested to delete all data", user.ID)
+
+	go func(user *models.User) {
+		logbuch.Info("deleting summaries for user '%s'", user.ID)
+		if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+			logbuch.Error("failed to clear summaries: %v", err)
+		}
+
+		logbuch.Info("deleting heartbeats for user '%s'", user.ID)
+		if err := h.heartbeatSrvc.DeleteByUser(user); err != nil {
+			logbuch.Error("failed to clear heartbeats: %v", err)
+		}
+	}(user)
+
+	return http.StatusAccepted, "deletion in progress, this may take a couple of seconds", ""
+}
+
 func (h *SettingsHandler) actionDeleteUser(w http.ResponseWriter, r *http.Request) (int, string, string) {
 	if h.config.IsDev() {
 		loadTemplates()

services/duration.go

@@ -22,7 +22,15 @@ func NewDurationService(heartbeatService IHeartbeatService) *DurationService {
 }
 
 func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *models.Filters) (models.Durations, error) {
-	heartbeats, err := srv.heartbeatService.GetAllWithin(from, to, user)
+	get := srv.heartbeatService.GetAllWithin
+
+	if filters != nil && !filters.IsEmpty() {
+		get = func(t1 time.Time, t2 time.Time, user *models.User) ([]*models.Heartbeat, error) {
+			return srv.heartbeatService.GetAllWithinByFilters(t1, t2, user, filters)
+		}
+	}
+
+	heartbeats, err := get(from, to, user)
 	if err != nil {
 		return nil, err
 	}

services/duration_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/muety/wakapi/mocks"
 	"github.com/muety/wakapi/models"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 	"math/rand"
 	"testing"
@@ -175,7 +176,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get_Filtered() {
 	)
 
 	from, to = suite.TestStartTime.Add(-1*time.Hour), suite.TestStartTime.Add(1*time.Hour)
-	suite.HeartbeatService.On("GetAllWithin", from, to, suite.TestUser).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
+	suite.HeartbeatService.On("GetAllWithinByFilters", from, to, suite.TestUser, mock.Anything).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
 
 	durations, err = sut.Get(from, to, suite.TestUser, models.NewFiltersWith(models.SummaryEditor, TestEditorGoland))
 	assert.Nil(suite.T(), err)

services/heartbeat.go

@@ -134,6 +134,14 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
 	return srv.augmented(heartbeats, user.ID)
 }
 
+func (srv *HeartbeatService) GetAllWithinByFilters(from, to time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	heartbeats, err := srv.repository.GetAllWithinByFilters(from, to, user, srv.filtersToColumnMap(filters))
+	if err != nil {
+		return nil, err
+	}
+	return srv.augmented(heartbeats, user.ID)
+}
+
 func (srv *HeartbeatService) GetLatestByUser(user *models.User) (*models.Heartbeat, error) {
 	return srv.repository.GetLatestByUser(user)
 }
@@ -171,9 +179,15 @@ func (srv *HeartbeatService) GetEntitySetByUser(entityType uint8, user *models.U
 }
 
 func (srv *HeartbeatService) DeleteBefore(t time.Time) error {
+	go srv.cache.Flush()
 	return srv.repository.DeleteBefore(t)
 }
 
+func (srv *HeartbeatService) DeleteByUser(user *models.User) error {
+	go srv.cache.Flush()
+	return srv.repository.DeleteByUser(user)
+}
+
 func (srv *HeartbeatService) augmented(heartbeats []*models.Heartbeat, userId string) ([]*models.Heartbeat, error) {
 	languageMapping, err := srv.languageMappingSrvc.ResolveByUser(userId)
 	if err != nil {
@@ -237,3 +251,14 @@ func (srv *HeartbeatService) countTotalCacheKey() string {
 func (srv *HeartbeatService) countCacheTtl() time.Duration {
 	return time.Duration(srv.config.App.CountCacheTTLMin) * time.Minute
 }
+
+func (srv *HeartbeatService) filtersToColumnMap(filters *models.Filters) map[string][]string {
+	columnMap := map[string][]string{}
+	for _, t := range models.SummaryTypes() {
+		f := filters.ResolveEntity(t)
+		if len(*f) > 0 {
+			columnMap[models.GetEntityColumn(t)] = *f
+		}
+	}
+	return columnMap
+}

services/mail/smtp.go

@@ -49,6 +49,11 @@ func (s *SMTPSendingService) Send(mail *models.Mail) error {
 		if ok, _ := c.Extension("AUTH"); !ok {
 			return errors.New("smtp: server doesn't support AUTH")
 		}
+
+		if len(s.config.Username) == 0 || len(s.config.Password) == 0 {
+			return errors.New("smtp: server requires authentication, but no authentication is provided")
+		}
+
 		if err = c.Auth(s.auth); err != nil {
 			return err
 		}

services/services.go

@@ -33,11 +33,13 @@ type IHeartbeatService interface {
 	CountByUser(*models.User) (int64, error)
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, *models.Filters) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsService interface {

static/assets/js/components/settings.js

@@ -21,6 +21,11 @@ PetiteVue.createApp({
             document.querySelector('#form-import-wakatime').submit()
         }
     },
+    confirmClearData() {
+        if (confirm('Are you sure? This can not be undone!')) {
+            document.querySelector('#form-clear-data').submit()
+        }
+    },
     confirmDeleteAccount() {
         if (confirm('Are you sure? This can not be undone!')) {
             document.querySelector('#form-delete-user').submit()

static/assets/js/summary.js

@@ -132,9 +132,9 @@ function draw(subselection) {
                 onClick: (event, data) => {
                     const idx = data[0].index
                     const name = wakapiData.projects[idx].key
-                    const query = new URLSearchParams(window.location.search)
-                    query.set('project', name)
-                    window.location.replace(`${window.location.pathname.slice(1)}?${query.toString()}`)
+                    const url = new URL(window.location.href)
+                    url.searchParams.set('project', name)
+                    window.location.href = url.href
                 },
                 onHover: (event, elem) => {
                     event.native.target.style.cursor = elem[0] ? 'pointer' : 'default'

version.txt

@@ -1 +1 @@
-2.2.2
+2.3.0

views/settings.tpl.html

@@ -609,7 +609,7 @@
                             Regenerate all pre-computed summaries from raw heartbeat data. This may be useful if, for some reason, summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively). This may take some time. Be careful and only run this action if you know, what your are doing, as data loss might occur.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmRegenerate">Clear and regenerate</button>
                     </div>
                 </form>
@@ -623,11 +623,25 @@
                             Please note that resetting your API key requires you to update your .wakatime.cfg files on all of your computers to make the WakaTime client send heartbeats again.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="submit" class="btn-danger ml-1">Reset API key</button>
                     </div>
                 </form>
 
+                <form action="" method="post" class="flex mb-8" id="form-clear-data">
+                    <input type="hidden" name="action" value="clear_data">
+
+                    <div class="w-1/2 mr-4 inline-block">
+                        <span class="font-semibold text-gray-300">Clear Data</span>
+                        <span class="block text-sm text-gray-600">
+                            Clear all your time tracking data from Wakapi. This cannot be undone. Be careful!
+                        </span>
+                    </div>
+                    <div class="w-1/2 ml-4 flex items-center">
+                        <button type="button" class="btn-danger ml-1" @click.stop="confirmClearData">Clear data</button>
+                    </div>
+                </form>
+
                 <form action="" method="post" class="flex mb-8" id="form-delete-user">
                     <input type="hidden" name="action" value="delete_account">
 
@@ -637,7 +651,7 @@
                             Deleting your account will cause all data, including all your heartbeats, to be erased from the server immediately. This action is irreversible. Be careful!
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmDeleteAccount">Delete account</button>
                     </div>
                 </form>

views/summary.tpl.html

@@ -189,12 +189,11 @@
                 # <strong>Step 1:</strong> Download WakaTime plugin for your IDE<br>
                 # See: https://wakatime.com/plugins<br><br>
 
-                # <strong>Step 2:</strong> Adapt your config<br>
-                $ vi ~/.wakatime.cfg<br>
-
+                # <strong>Step 2:</strong> Set your ~/.wakatime.cfg to this:<br><br>
                 <!-- https://github.com/muety/wakapi/issues/224#issuecomment-890855563 -->
-                # Set <em>api_url = <span class="with-url-inner">%s/api</span></em><br>
-                # Set <em>api_key = <span id="api-key-instruction">{{ .ApiKey }}</span></em><br><br>
+                [settings]<br>
+                api_url = <span class="with-url-inner">%s/api</span><br>
+                api_key = <span id="api-key-instruction">{{ .ApiKey }}</span><br><br>
 
                 # <strong>Step 3:</strong> Start coding and then check back here!
             </div>