feat: implement data retention mechanism

services/heartbeat.go

@@ -192,6 +192,11 @@ func (srv *HeartbeatService) DeleteByUser(user *models.User) error {
 	return srv.repository.DeleteByUser(user)
 }
 
+func (srv *HeartbeatService) DeleteByUserBefore(user *models.User, t time.Time) error {
+	go srv.cache.Flush()
+	return srv.repository.DeleteByUserBefore(user, t)
+}
+
 func (srv *HeartbeatService) augmented(heartbeats []*models.Heartbeat, userId string) ([]*models.Heartbeat, error) {
 	languageMapping, err := srv.languageMappingSrvc.ResolveByUser(userId)
 	if err != nil {

services/housekeeping.go

@@ -0,0 +1,81 @@
+package services
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/artifex/v2"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	"time"
+)
+
+type HousekeepingService struct {
+	config        *config.Config
+	userSrvc      IUserService
+	heartbeatSrvc IHeartbeatService
+	summarySrvc   ISummaryService
+	queueDefault  *artifex.Dispatcher
+	queueWorkers  *artifex.Dispatcher
+}
+
+func NewHousekeepingService(userService IUserService, heartbeatService IHeartbeatService, summaryService ISummaryService) *HousekeepingService {
+	return &HousekeepingService{
+		config:        config.Get(),
+		userSrvc:      userService,
+		heartbeatSrvc: heartbeatService,
+		summarySrvc:   summaryService,
+		queueDefault:  config.GetDefaultQueue(),
+		queueWorkers:  config.GetQueue(config.QueueHousekeeping),
+	}
+}
+
+func (s *HousekeepingService) Schedule() {
+	if s.config.App.DataRetentionMonths <= 0 {
+		return
+	}
+
+	logbuch.Info("scheduling data cleanup")
+
+	// this is not exactly precise, because of summer / winter time, etc.
+	retentionDuration := time.Now().Sub(time.Now().AddDate(0, -s.config.App.DataRetentionMonths, 0))
+
+	_, err := s.queueDefault.DispatchCron(func() {
+		// fetch all users
+		users, err := s.userSrvc.GetAll()
+		if err != nil {
+			config.Log().Error("failed to get users for data cleanup, %v", err)
+			return
+		}
+
+		// schedule jobs
+		for _, u := range users {
+			user := *u
+			s.queueWorkers.Dispatch(func() {
+				if err := s.ClearOldUserData(&user, retentionDuration); err != nil {
+					config.Log().Error("failed to clear old user data for '%s'", user.ID)
+				}
+			})
+		}
+	}, s.config.App.DataCleanupTime)
+
+	if err != nil {
+		config.Log().Error("failed to dispatch data cleanup jobs, %v", err)
+	}
+}
+
+func (s *HousekeepingService) ClearOldUserData(user *models.User, maxAge time.Duration) error {
+	before := time.Now().Add(-maxAge)
+	logbuch.Warn("cleaning up user data for '%s' older than %v", user.ID, before)
+
+	// clear old heartbeats
+	if err := s.heartbeatSrvc.DeleteByUserBefore(user, before); err != nil {
+		return err
+	}
+
+	// clear old summaries
+	logbuch.Info("clearing summaries for user '%s' older than %v", user.ID, before)
+	if err := s.summarySrvc.DeleteByUserBefore(user.ID, before); err != nil {
+		return err
+	}
+
+	return nil
+}

services/services.go

@@ -42,6 +42,7 @@ type IHeartbeatService interface {
 	GetEntitySetByUser(uint8, *models.User) ([]string, error)
 	DeleteBefore(time.Time) error
 	DeleteByUser(*models.User) error
+	DeleteByUserBefore(*models.User, time.Time) error
 }
 
 type IDiagnosticsService interface {
@@ -89,6 +90,7 @@ type ISummaryService interface {
 	Summarize(time.Time, time.Time, *models.User, *models.Filters) (*models.Summary, error)
 	GetLatestByUser() ([]*models.TimeByUser, error)
 	DeleteByUser(string) error
+	DeleteByUserBefore(string, time.Time) error
 	Insert(*models.Summary) error
 }
 
@@ -97,6 +99,11 @@ type IReportService interface {
 	SendReport(*models.User, time.Duration) error
 }
 
+type IHousekeepingService interface {
+	Schedule()
+	ClearOldUserData(*models.User, time.Duration) error
+}
+
 type ILeaderboardService interface {
 	Schedule()
 	ComputeLeaderboard([]*models.User, *models.IntervalKey, []uint8) error

services/summary.go

@@ -208,6 +208,11 @@ func (srv *SummaryService) DeleteByUser(userId string) error {
 	return srv.repository.DeleteByUser(userId)
 }
 
+func (srv *SummaryService) DeleteByUserBefore(userId string, t time.Time) error {
+	srv.invalidateUserCache(userId)
+	return srv.repository.DeleteByUserBefore(userId, t)
+}
+
 func (srv *SummaryService) Insert(summary *models.Summary) error {
 	srv.invalidateUserCache(summary.UserID)
 	return srv.repository.Insert(summary)