feat: ability to clear all user data (resolve #339)

README.md

@@ -128,6 +128,11 @@ You can specify configuration options either via a config file (default: `config
 | YAML Key / Env. Variable                                                     | Default                                          | Description                                                                                                                                                              |
 |------------------------------------------------------------------------------|--------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `env` /<br>`ENVIRONMENT`                                                     | `dev`                                            | Whether to use development- or production settings                                                                                                                       |
+| `app.aggregation_time`<br>`WAKAPI_AGGREGATION_TIME`                          | `02:15`                                          | Time of day at which to periodically run summary generation for all users                                                                                                |
+| `app.report_time_weekly`<br>`WAKAPI_REPORT_TIME_WEEKLY`                      | `fri,18:00`                                      | Week day and time at which to send e-mail reports                                                                                                                        |
+| `app.import_batch_size`<br>`WAKAPI_IMPORT_BATCH_SIZE`                        | `50`                                             | Size of batches of heartbeats to insert to the database during importing from external services                                                                          |
+| `app.inactive_days`<br>`WAKAPI_INACTIVE_DAYS`                                | `7`                                              | Number of days after which to consider a user inactive (only for metrics)                                                                                                |
+| `app.heartbeat_max_age`<br>`WAKAPI_HEARTBEAT_MAX_AGE`                        | `4320h`                                          | Maximum acceptable age of a heartbeat (see [`ParseDuration`](https://pkg.go.dev/time#ParseDuration))                                                                     |
 | `app.custom_languages`                                                       | -                                                | Map from file endings to language names                                                                                                                                  |
 | `app.avatar_url_template`                                                    | (see [`config.default.yml`](config.default.yml)) | URL template for external user avatar images (e.g. from [Dicebear](https://dicebear.com) or [Gravatar](https://gravatar.com))                                            |
 | `server.port` /<br> `WAKAPI_PORT`                                            | `3000`                                           | Port to listen on                                                                                                                                                        |

config.default.yml

@@ -16,6 +16,7 @@ app:
   report_time_weekly: 'fri,18:00'     # time at which to fan out weekly reports (format: '<weekday)>,<daytime>')
   inactive_days: 7                    # time of previous days within a user must have logged in to be considered active
   import_batch_size: 50               # maximum number of heartbeats to insert into the database within one transaction
+  heartbeat_max_age: '4320h'          # maximum acceptable age of a heartbeat (see https://pkg.go.dev/time#ParseDuration)
   custom_languages:
     vue: Vue
     jsx: JSX

config/config.go

@@ -68,6 +68,7 @@ type appConfig struct {
 	ImportBackoffMin  int                          `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"`
 	ImportBatchSize   int                          `yaml:"import_batch_size" default:"50" env:"WAKAPI_IMPORT_BATCH_SIZE"`
 	InactiveDays      int                          `yaml:"inactive_days" default:"7" env:"WAKAPI_INACTIVE_DAYS"`
+	HeartbeatMaxAge   string                       `yaml:"heartbeat_max_age" default:"4320h" env:"WAKAPI_HEARTBEAT_MAX_AGE"`
 	CountCacheTTLMin  int                          `yaml:"count_cache_ttl_min" default:"30" env:"WAKAPI_COUNT_CACHE_TTL_MIN"`
 	AvatarURLTemplate string                       `yaml:"avatar_url_template" default:"api/avatar/{username_hash}.svg"`
 	CustomLanguages   map[string]string            `yaml:"custom_languages"`
@@ -242,6 +243,11 @@ func (c *appConfig) GetWeeklyReportTime() string {
 	return strings.Split(c.ReportTimeWeekly, ",")[1]
 }
 
+func (c *appConfig) HeartbeatsMaxAge() time.Duration {
+	d, _ := time.ParseDuration(c.HeartbeatMaxAge)
+	return d
+}
+
 func (c *dbConfig) IsSQLite() bool {
 	return c.Dialect == "sqlite3"
 }
@@ -400,6 +406,9 @@ func Load(version string) *Config {
 	if _, err := time.Parse("15:04", config.App.AggregationTime); err != nil {
 		logbuch.Fatal("invalid interval set for aggregation_time")
 	}
+	if _, err := time.ParseDuration(config.App.HeartbeatMaxAge); err != nil {
+		logbuch.Fatal("invalid duration set for heartbeat_max_age")
+	}
 
 	Set(config)
 	return Get()

migrations/20220313_index_generation_hint.go

@@ -1,24 +0,0 @@
-package migrations
-
-import (
-	"github.com/emvi/logbuch"
-	"github.com/muety/wakapi/config"
-	"gorm.io/gorm"
-)
-
-func init() {
-	const name = "20220313-index_generation_hint"
-	f := migrationFunc{
-		name: name,
-		f: func(db *gorm.DB, cfg *config.Config) error {
-			if hasRun(name, db) {
-				return nil
-			}
-			logbuch.Info("please note: the following migrations might take a few minutes, as column types are changed and new indexes are created, have some patience")
-			setHasRun(name, db)
-			return nil
-		},
-	}
-
-	registerPreMigration(f)
-}

migrations/20220317_align_num_heartbeats.go

@@ -0,0 +1,56 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220317-align_num_heartbeats"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+
+			logbuch.Info("this may take a while!")
+
+			// find all summaries whose num_heartbeats is zero even though they have items
+			var faultyIds []uint
+
+			if err := db.Model(&models.Summary{}).
+				Distinct("summaries.id").
+				Joins("INNER JOIN summary_items ON summaries.num_heartbeats = 0 AND summaries.id = summary_items.summary_id").
+				Scan(&faultyIds).Error; err != nil {
+				return err
+			}
+
+			// update their heartbeats counter
+			result := db.
+				Table("summaries AS s1").
+				Where("s1.id IN ?", faultyIds).
+				Update(
+					"num_heartbeats",
+					db.
+						Model(&models.Heartbeat{}).
+						Select("COUNT(*)").
+						Where("user_id = ?", gorm.Expr("s1.user_id")).
+						Where("time BETWEEN ? AND ?", gorm.Expr("s1.from_time"), gorm.Expr("s1.to_time")),
+				)
+
+			if err := result.Error; err != nil {
+				return err
+			}
+
+			logbuch.Info("corrected heartbeats counter of %d summaries", result.RowsAffected)
+
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPostMigration(f)
+}

mocks/heartbeat_service.go

@@ -69,3 +69,8 @@ func (m *HeartbeatServiceMock) DeleteBefore(time time.Time) error {
 	args := m.Called(time)
 	return args.Error(0)
 }
+
+func (m *HeartbeatServiceMock) DeleteByUser(u *models.User) error {
+	args := m.Called(u)
+	return args.Error(0)
+}

models/heartbeat.go

@@ -34,6 +34,11 @@ func (h *Heartbeat) Valid() bool {
 	return h.User != nil && h.UserID != "" && h.User.ID == h.UserID && h.Time != CustomTime(time.Time{})
 }
 
+func (h *Heartbeat) Timely(maxAge time.Duration) bool {
+	now := time.Now()
+	return now.Sub(h.Time.T()) <= maxAge && h.Time.T().Before(now)
+}
+
 func (h *Heartbeat) Augment(languageMappings map[string]string) {
 	maxPrec := -1 // precision / mapping complexity -> more concrete ones shall take precedence
 	for ending, value := range languageMappings {

models/summary.go

@@ -101,7 +101,23 @@ func (s *Summary) MappedItems() map[uint8]*SummaryItems {
 }
 
 func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
-	return s.MappedItems()[summaryType]
+	switch summaryType {
+	case SummaryProject:
+		return &s.Projects
+	case SummaryLanguage:
+		return &s.Languages
+	case SummaryEditor:
+		return &s.Editors
+	case SummaryOS:
+		return &s.OperatingSystems
+	case SummaryMachine:
+		return &s.Machines
+	case SummaryLabel:
+		return &s.Labels
+	case SummaryBranch:
+		return &s.Branches
+	}
+	return nil
 }
 
 func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {

repositories/heartbeat.go

@@ -176,3 +176,12 @@ func (r *HeartbeatRepository) DeleteBefore(t time.Time) error {
 	}
 	return nil
 }
+
+func (r *HeartbeatRepository) DeleteByUser(user *models.User) error {
+	if err := r.db.
+		Where("user_id = ?", user.ID).
+		Delete(models.Heartbeat{}).Error; err != nil {
+		return err
+	}
+	return nil
+}

repositories/repositories.go

@@ -30,6 +30,7 @@ type IHeartbeatRepository interface {
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsRepository interface {

repositories/summary.go

@@ -18,15 +18,15 @@ func (r *SummaryRepository) GetAll() ([]*models.Summary, error) {
 	var summaries []*models.Summary
 	if err := r.db.
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -44,15 +44,15 @@ func (r *SummaryRepository) GetByUserWithin(user *models.User, from, to time.Tim
 		Where("from_time >= ?", from.Local()).
 		Where("to_time <= ?", to.Local()).
 		Order("from_time asc").
-		Preload("Projects", "type = ?", models.SummaryProject).
-		Preload("Languages", "type = ?", models.SummaryLanguage).
-		Preload("Editors", "type = ?", models.SummaryEditor).
-		Preload("OperatingSystems", "type = ?", models.SummaryOS).
-		Preload("Machines", "type = ?", models.SummaryMachine).
 		// branch summaries are currently not persisted, as only relevant in combination with project filter
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
+
+	if err := r.populateItems(summaries); err != nil {
+		return nil, err
+	}
+
 	return summaries, nil
 }
 
@@ -74,3 +74,32 @@ func (r *SummaryRepository) DeleteByUser(userId string) error {
 	}
 	return nil
 }
+
+// inplace
+func (r *SummaryRepository) populateItems(summaries []*models.Summary) error {
+	summaryMap := map[uint]*models.Summary{}
+	summaryIds := make([]uint, len(summaries))
+	for i, s := range summaries {
+		if s.NumHeartbeats == 0 {
+			continue
+		}
+		summaryMap[s.ID] = s
+		summaryIds[i] = s.ID
+	}
+
+	var items []*models.SummaryItem
+
+	if err := r.db.
+		Model(&models.SummaryItem{}).
+		Where("summary_id in ?", summaryIds).
+		Find(&items).Error; err != nil {
+		return err
+	}
+
+	for _, item := range items {
+		l := summaryMap[item.SummaryID].ItemsByType(item.Type)
+		*l = append(*l, item)
+	}
+
+	return nil
+}

routes/api/heartbeat.go

@@ -86,7 +86,7 @@ func (h *HeartbeatApiHandler) Post(w http.ResponseWriter, r *http.Request) {
 		hb.UserID = user.ID
 		hb.UserAgent = userAgent
 
-		if !hb.Valid() {
+		if !hb.Valid() || !hb.Timely(h.config.App.HeartbeatsMaxAge()) {
 			w.WriteHeader(http.StatusBadRequest)
 			w.Write([]byte("invalid heartbeat object"))
 			return

routes/settings.go

@@ -151,6 +151,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
 		return h.actionImportWakatime
 	case "regenerate_summaries":
 		return h.actionRegenerateSummaries
+	case "clear_data":
+		return h.actionClearData
 	case "delete_account":
 		return h.actionDeleteUser
 	}
@@ -553,6 +555,29 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
 	return http.StatusAccepted, "summaries are being regenerated - this may take a up to a couple of minutes, please come back later", ""
 }
 
+func (h *SettingsHandler) actionClearData(w http.ResponseWriter, r *http.Request) (int, string, string) {
+	if h.config.IsDev() {
+		loadTemplates()
+	}
+
+	user := middlewares.GetPrincipal(r)
+	logbuch.Info("user '%s' requested to delete all data", user.ID)
+
+	go func(user *models.User) {
+		logbuch.Info("deleting summaries for user '%s'", user.ID)
+		if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+			logbuch.Error("failed to clear summaries: %v", err)
+		}
+
+		logbuch.Info("deleting heartbeats for user '%s'", user.ID)
+		if err := h.heartbeatSrvc.DeleteByUser(user); err != nil {
+			logbuch.Error("failed to clear heartbeats: %v", err)
+		}
+	}(user)
+
+	return http.StatusAccepted, "deletion in progress, this may take a couple of seconds", ""
+}
+
 func (h *SettingsHandler) actionDeleteUser(w http.ResponseWriter, r *http.Request) (int, string, string) {
 	if h.config.IsDev() {
 		loadTemplates()

services/heartbeat.go

@@ -179,9 +179,15 @@ func (srv *HeartbeatService) GetEntitySetByUser(entityType uint8, user *models.U
 }
 
 func (srv *HeartbeatService) DeleteBefore(t time.Time) error {
+	go srv.cache.Flush()
 	return srv.repository.DeleteBefore(t)
 }
 
+func (srv *HeartbeatService) DeleteByUser(user *models.User) error {
+	go srv.cache.Flush()
+	return srv.repository.DeleteByUser(user)
+}
+
 func (srv *HeartbeatService) augmented(heartbeats []*models.Heartbeat, userId string) ([]*models.Heartbeat, error) {
 	languageMapping, err := srv.languageMappingSrvc.ResolveByUser(userId)
 	if err != nil {

services/services.go

@@ -39,6 +39,7 @@ type IHeartbeatService interface {
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
+	DeleteByUser(*models.User) error
 }
 
 type IDiagnosticsService interface {

static/assets/js/components/settings.js

@@ -21,6 +21,11 @@ PetiteVue.createApp({
             document.querySelector('#form-import-wakatime').submit()
         }
     },
+    confirmClearData() {
+        if (confirm('Are you sure? This can not be undone!')) {
+            document.querySelector('#form-clear-data').submit()
+        }
+    },
     confirmDeleteAccount() {
         if (confirm('Are you sure? This can not be undone!')) {
             document.querySelector('#form-delete-user').submit()

version.txt

@@ -1 +1 @@
-2.2.5
+2.3.0

views/settings.tpl.html

@@ -609,7 +609,7 @@
                             Regenerate all pre-computed summaries from raw heartbeat data. This may be useful if, for some reason, summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively). This may take some time. Be careful and only run this action if you know, what your are doing, as data loss might occur.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmRegenerate">Clear and regenerate</button>
                     </div>
                 </form>
@@ -623,11 +623,25 @@
                             Please note that resetting your API key requires you to update your .wakatime.cfg files on all of your computers to make the WakaTime client send heartbeats again.
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="submit" class="btn-danger ml-1">Reset API key</button>
                     </div>
                 </form>
 
+                <form action="" method="post" class="flex mb-8" id="form-clear-data">
+                    <input type="hidden" name="action" value="clear_data">
+
+                    <div class="w-1/2 mr-4 inline-block">
+                        <span class="font-semibold text-gray-300">Clear Data</span>
+                        <span class="block text-sm text-gray-600">
+                            Clear all your time tracking data from Wakapi. This cannot be undone. Be careful!
+                        </span>
+                    </div>
+                    <div class="w-1/2 ml-4 flex items-center">
+                        <button type="button" class="btn-danger ml-1" @click.stop="confirmClearData">Clear data</button>
+                    </div>
+                </form>
+
                 <form action="" method="post" class="flex mb-8" id="form-delete-user">
                     <input type="hidden" name="action" value="delete_account">
 
@@ -637,7 +651,7 @@
                             Deleting your account will cause all data, including all your heartbeats, to be erased from the server immediately. This action is irreversible. Be careful!
                         </span>
                     </div>
-                    <div class="w-1/2 ml-4">
+                    <div class="w-1/2 ml-4 flex items-center">
                         <button type="button" class="btn-danger ml-1" @click.stop="confirmDeleteAccount">Delete account</button>
                     </div>
                 </form>