chore: optimize import date range

README.md

@@ -68,7 +68,7 @@ I'd love to get some community feedback from active Wakapi users. If you want, p
 Plans for the near future mainly include, besides usual improvements and bug fixes, a UI redesign as well as additional types of charts and statistics (see [#101](https://github.com/muety/wakapi/issues/101), [#80](https://github.com/muety/wakapi/issues/80), [#76](https://github.com/muety/wakapi/issues/76), [#12](https://github.com/muety/wakapi/issues/12)). If you have feature requests or any kind of improvement proposals feel free to open an issue or share them in our [user survey](https://github.com/muety/wakapi/issues/82). 
 
 ## ⌨️ How to use?
-There are different options for how to use Wakapi, ranging from out hosted cloud service to self-hosting it. Regardless of which option choose, you will always have to do the [client setup](#-client-setup) in addition. 
+There are different options for how to use Wakapi, ranging from our hosted cloud service to self-hosting it. Regardless of which option choose, you will always have to do the [client setup](#-client-setup) in addition. 
 
 ### ☁️ Option 1: Use [wakapi.dev](https://wakapi.dev)
 If you want to you out free, hosted cloud service, all you need to do is create an account and the set up your client-side tooling (see below).

models/heartbeat.go

@@ -25,6 +25,7 @@ type Heartbeat struct {
 	Time            CustomTime     `json:"time" gorm:"type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time,idx_time_user"`
 	Hash            string         `json:"-" gorm:"type:varchar(17); uniqueIndex"`
 	Origin          string         `json:"-" hash:"ignore"`
+	OriginId        string         `json:"-" hash:"ignore"`
 	languageRegex   *regexp.Regexp `hash:"ignore"`
 }
 

repositories/heartbeart.go

@@ -37,6 +37,21 @@ func (r *HeartbeatRepository) CountByUser(user *models.User) (int64, error) {
 	return count, nil
 }
 
+func (r *HeartbeatRepository) GetLatestByOriginAndUser(origin string, user *models.User) (*models.Heartbeat, error) {
+	var heartbeat models.Heartbeat
+	if err := r.db.
+		Model(&models.Heartbeat{}).
+		Where(&models.Heartbeat{
+			UserID: user.ID,
+			Origin: origin,
+		}).
+		Order("time desc").
+		First(&heartbeat).Error; err != nil {
+		return nil, err
+	}
+	return &heartbeat, nil
+}
+
 func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
 	var heartbeats []*models.Heartbeat
 	if err := r.db.

repositories/repositories.go

@@ -20,6 +20,7 @@ type IHeartbeatRepository interface {
 	CountByUser(*models.User) (int64, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
+	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
 	DeleteBefore(time.Time) error
 }
 

routes/settings.go

@@ -362,10 +362,18 @@ func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Req
 			println(err)
 		}
 
+		var stream <-chan *models.Heartbeat
+		if latest, err := h.heartbeatSrvc.GetLatestByOriginAndUser(imports.OriginWakatime, user); latest == nil || err != nil {
+			stream = importer.ImportAll(user)
+		} else {
+			// if an import has happened before, only import heartbeats newer than the latest of the last import
+			stream = importer.Import(user, latest.Time.T(), time.Now())
+		}
+
 		count := 0
 		batch := make([]*models.Heartbeat, 0)
 
-		for hb := range importer.Import(user) {
+		for hb := range stream {
 			count++
 			batch = append(batch, hb)
 
@@ -389,7 +397,7 @@ func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Req
 		Value: time.Now().Format(time.RFC822),
 	})
 
-	return http.StatusAccepted, "Import started. This may take a few minutes.", ""
+	return http.StatusAccepted, "ImportAll started. This may take a few minutes.", ""
 }
 
 func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *http.Request) (int, string, string) {

services/heartbeat.go

@@ -42,6 +42,10 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
 	return srv.augmented(heartbeats, user.ID)
 }
 
+func (srv *HeartbeatService) GetLatestByOriginAndUser(origin string, user *models.User) (*models.Heartbeat, error) {
+	return srv.repository.GetLatestByOriginAndUser(origin, user)
+}
+
 func (srv *HeartbeatService) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	return srv.repository.GetFirstByUsers()
 }

services/imports/importers.go

@@ -1,7 +1,11 @@
 package imports
 
-import "github.com/muety/wakapi/models"
+import (
+	"github.com/muety/wakapi/models"
+	"time"
+)
 
 type HeartbeatImporter interface {
-	Import(*models.User) <-chan *models.Heartbeat
+	Import(*models.User, time.Time, time.Time) <-chan *models.Heartbeat
+	ImportAll(*models.User) <-chan *models.Heartbeat
 }

services/imports/wakatime.go

@@ -17,9 +17,8 @@ import (
 	"time"
 )
 
-const (
+const OriginWakatime = "wakatime"
-	maxWorkers = 6
+const maxWorkers = 6
-)
 
 type WakatimeHeartbeatImporter struct {
 	ApiKey string
@@ -31,7 +30,7 @@ func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
 	}
 }
 
-func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Heartbeat {
+func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time, maxTo time.Time) <-chan *models.Heartbeat {
 	out := make(chan *models.Heartbeat)
 
 	go func(user *models.User, out chan *models.Heartbeat) {
@@ -41,6 +40,13 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Hea
 			return
 		}
 
+		if startDate.Before(minFrom) {
+			startDate = minFrom
+		}
+		if endDate.After(maxTo) {
+			endDate = maxTo
+		}
+
 		userAgents, err := w.fetchUserAgents()
 		if err != nil {
 			logbuch.Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
@@ -82,6 +88,10 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Hea
 	return out
 }
 
+func (w *WakatimeHeartbeatImporter) ImportAll(user *models.User) <-chan *models.Heartbeat {
+	return w.Import(user, time.Time{}, time.Now())
+}
+
 // https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
 func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
 	httpClient := &http.Client{Timeout: 10 * time.Second}
@@ -211,7 +221,8 @@ func mapHeartbeat(
 		OperatingSystem: ua.Os,
 		Machine:         entry.MachineNameId, // TODO
 		Time:            entry.Time,
-		Origin:          fmt.Sprintf("wt@%s", entry.Id),
+		Origin:          OriginWakatime,
+		OriginId:        entry.Id,
 	}).Hashed()
 }
 

services/services.go

@@ -31,6 +31,7 @@ type IHeartbeatService interface {
 	CountByUser(*models.User) (int64, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
+	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
 	DeleteBefore(time.Time) error
 }
 

views/settings.tpl.html

@@ -474,7 +474,9 @@
     const btnImportWakatime = document.querySelector('#btn-import-wakatime')
     const formImportWakatime = document.querySelector('#form-import-wakatime')
     btnImportWakatime.addEventListener('click', () => {
+        if (confirm('Are you sure? The import can not be undone.')) {
             formImportWakatime.submit()
+        }
     })
 </script>