feat: wakatime data import (resolve #87)

config/config.go

@@ -29,12 +29,16 @@ const (
 
 	KeyLatestTotalTime  = "latest_total_time"
 	KeyLatestTotalUsers = "latest_total_users"
+	KeyLastImportImport = "last_import"
 )
 
 const (
 	WakatimeApiUrl               = "https://wakatime.com/api/v1"
-	WakatimeApiHeartbeatsEndpoint = "/users/current/heartbeats.bulk"
-	WakatimeApiUserEndpoint       = "/users/current"
+	WakatimeApiUserUrl           = "/users/current"
+	WakatimeApiAllTimeUrl        = "/users/current/all_time_since_today"
+	WakatimeApiHeartbeatsUrl     = "/users/current/heartbeats"
+	WakatimeApiHeartbeatsBulkUrl = "/users/current/heartbeats.bulk"
+	WakatimeApiUserAgentsUrl     = "/users/current/user_agents"
 )
 
 var cfg *Config
@@ -43,6 +47,8 @@ var cFlag = flag.String("config", defaultConfigPath, "config file location")
 type appConfig struct {
 	AggregationTime  string                       `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"`
 	CountingTime     string                       `yaml:"counting_time" default:"05:15" env:"WAKAPI_COUNTING_TIME"`
+	ImportBackoffMin int                          `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"`
+	ImportBatchSize  int                          `yaml:"import_batch_size" default:"25" env:"WAKAPI_IMPORT_BATCH_SIZE"`
 	CustomLanguages  map[string]string            `yaml:"custom_languages"`
 	Colors           map[string]map[string]string `yaml:"-"`
 }

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
 	gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
 	gopkg.in/yaml.v2 v2.2.8 // indirect
 	gorm.io/driver/mysql v1.0.3

main.go

@@ -136,7 +136,7 @@ func main() {
 
 	// MVC Handlers
 	summaryHandler := routes.NewSummaryHandler(summaryService, userService)
-	settingsHandler := routes.NewSettingsHandler(userService, summaryService, aliasService, aggregationService, languageMappingService)
+	settingsHandler := routes.NewSettingsHandler(userService, heartbeatService, summaryService, aliasService, aggregationService, languageMappingService, keyValueService)
 	homeHandler := routes.NewHomeHandler(keyValueService)
 	loginHandler := routes.NewLoginHandler(userService)
 	imprintHandler := routes.NewImprintHandler(keyValueService)

middlewares/custom/wakatime.go

@@ -63,7 +63,7 @@ func (m *WakatimeRelayMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Reque
 
 	go m.send(
 		http.MethodPost,
-		config.WakatimeApiUrl+config.WakatimeApiHeartbeatsEndpoint,
+		config.WakatimeApiUrl+config.WakatimeApiHeartbeatsBulkUrl,
 		bytes.NewReader(body),
 		headers,
 	)

models/compat/wakatime/v1/all_time.go

@@ -9,10 +9,10 @@ import (
 // https://wakatime.com/developers#all_time_since_today
 
 type AllTimeViewModel struct {
-	Data *allTimeData `json:"data"`
+	Data *AllTimeData `json:"data"`
 }
 
-type allTimeData struct {
+type AllTimeData struct {
 	TotalSeconds float32 `json:"total_seconds"` // total number of seconds logged since account created
 	Text         string  `json:"text"`          // total time logged since account created as human readable string>
 	IsUpToDate   bool    `json:"is_up_to_date"` // true if the stats are up to date; when false, a 202 response code is returned and stats will be refreshed soon>
@@ -27,7 +27,7 @@ func NewAllTimeFrom(summary *models.Summary, filters *models.Filters) *AllTimeVi
 	}
 
 	return &AllTimeViewModel{
-		Data: &allTimeData{
+		Data: &AllTimeData{
 			TotalSeconds: float32(total.Seconds()),
 			Text:         utils.FmtWakatimeDuration(total),
 			IsUpToDate:   true,

models/compat/wakatime/v1/heartbeat.go

@@ -0,0 +1,25 @@
+package v1
+
+import "github.com/muety/wakapi/models"
+
+type HeartbeatsViewModel struct {
+	Data []*HeartbeatEntry `json:"data"`
+}
+
+// Incomplete, for now, only the subset of fields is implemented
+// that is actually required for the import
+
+type HeartbeatEntry struct {
+	Id            string
+	Branch        string
+	Category      string
+	Entity        string
+	IsWrite       bool `json:"is_write"`
+	Language      string
+	Project       string
+	Time          models.CustomTime
+	Type          string
+	UserId        string `json:"user_id"`
+	MachineNameId string `json:"machine_name_id"`
+	UserAgentId   string `json:"user_agent_id"`
+}

models/compat/wakatime/v1/summaries.go

@@ -13,24 +13,24 @@ import (
 // https://pastr.de/v/736450
 
 type SummariesViewModel struct {
-	Data  []*summariesData `json:"data"`
+	Data  []*SummariesData `json:"data"`
 	End   time.Time        `json:"end"`
 	Start time.Time        `json:"start"`
 }
 
-type summariesData struct {
-	Categories       []*summariesEntry    `json:"categories"`
-	Dependencies     []*summariesEntry    `json:"dependencies"`
-	Editors          []*summariesEntry    `json:"editors"`
-	Languages        []*summariesEntry    `json:"languages"`
-	Machines         []*summariesEntry    `json:"machines"`
-	OperatingSystems []*summariesEntry    `json:"operating_systems"`
-	Projects         []*summariesEntry    `json:"projects"`
-	GrandTotal       *summariesGrandTotal `json:"grand_total"`
-	Range            *summariesRange      `json:"range"`
+type SummariesData struct {
+	Categories       []*SummariesEntry    `json:"categories"`
+	Dependencies     []*SummariesEntry    `json:"dependencies"`
+	Editors          []*SummariesEntry    `json:"editors"`
+	Languages        []*SummariesEntry    `json:"languages"`
+	Machines         []*SummariesEntry    `json:"machines"`
+	OperatingSystems []*SummariesEntry    `json:"operating_systems"`
+	Projects         []*SummariesEntry    `json:"projects"`
+	GrandTotal       *SummariesGrandTotal `json:"grand_total"`
+	Range            *SummariesRange      `json:"range"`
 }
 
-type summariesEntry struct {
+type SummariesEntry struct {
 	Digital      string  `json:"digital"`
 	Hours        int     `json:"hours"`
 	Minutes      int     `json:"minutes"`
@@ -41,7 +41,7 @@ type summariesEntry struct {
 	TotalSeconds float64 `json:"total_seconds"`
 }
 
-type summariesGrandTotal struct {
+type SummariesGrandTotal struct {
 	Digital      string  `json:"digital"`
 	Hours        int     `json:"hours"`
 	Minutes      int     `json:"minutes"`
@@ -49,7 +49,7 @@ type summariesGrandTotal struct {
 	TotalSeconds float64 `json:"total_seconds"`
 }
 
-type summariesRange struct {
+type SummariesRange struct {
 	Date     string    `json:"date"`
 	End      time.Time `json:"end"`
 	Start    time.Time `json:"start"`
@@ -58,7 +58,7 @@ type summariesRange struct {
 }
 
 func NewSummariesFrom(summaries []*models.Summary, filters *models.Filters) *SummariesViewModel {
-	data := make([]*summariesData, len(summaries))
+	data := make([]*SummariesData, len(summaries))
 	minDate, maxDate := time.Now().Add(1*time.Second), time.Time{}
 
 	for i, s := range summaries {
@@ -79,27 +79,27 @@ func NewSummariesFrom(summaries []*models.Summary, filters *models.Filters) *Sum
 	}
 }
 
-func newDataFrom(s *models.Summary) *summariesData {
+func newDataFrom(s *models.Summary) *SummariesData {
 	zone, _ := time.Now().Zone()
 	total := s.TotalTime()
 	totalHrs, totalMins := int(total.Hours()), int((total - time.Duration(total.Hours())*time.Hour).Minutes())
 
-	data := &summariesData{
-		Categories:       make([]*summariesEntry, 0),
-		Dependencies:     make([]*summariesEntry, 0),
-		Editors:          make([]*summariesEntry, len(s.Editors)),
-		Languages:        make([]*summariesEntry, len(s.Languages)),
-		Machines:         make([]*summariesEntry, len(s.Machines)),
-		OperatingSystems: make([]*summariesEntry, len(s.OperatingSystems)),
-		Projects:         make([]*summariesEntry, len(s.Projects)),
-		GrandTotal: &summariesGrandTotal{
+	data := &SummariesData{
+		Categories:       make([]*SummariesEntry, 0),
+		Dependencies:     make([]*SummariesEntry, 0),
+		Editors:          make([]*SummariesEntry, len(s.Editors)),
+		Languages:        make([]*SummariesEntry, len(s.Languages)),
+		Machines:         make([]*SummariesEntry, len(s.Machines)),
+		OperatingSystems: make([]*SummariesEntry, len(s.OperatingSystems)),
+		Projects:         make([]*SummariesEntry, len(s.Projects)),
+		GrandTotal: &SummariesGrandTotal{
 			Digital:      fmt.Sprintf("%d:%d", totalHrs, totalMins),
 			Hours:        totalHrs,
 			Minutes:      totalMins,
 			Text:         utils.FmtWakatimeDuration(total),
 			TotalSeconds: total.Seconds(),
 		},
-		Range: &summariesRange{
+		Range: &SummariesRange{
 			Date:     time.Now().Format(time.RFC3339),
 			End:      s.ToTime.T(),
 			Start:    s.FromTime.T(),
@@ -111,21 +111,21 @@ func newDataFrom(s *models.Summary) *summariesData {
 	var wg sync.WaitGroup
 	wg.Add(5)
 
-	go func(data *summariesData) {
+	go func(data *SummariesData) {
 		defer wg.Done()
 		for i, e := range s.Projects {
 			data.Projects[i] = convertEntry(e, s.TotalTimeBy(models.SummaryProject))
 		}
 	}(data)
 
-	go func(data *summariesData) {
+	go func(data *SummariesData) {
 		defer wg.Done()
 		for i, e := range s.Editors {
 			data.Editors[i] = convertEntry(e, s.TotalTimeBy(models.SummaryEditor))
 		}
 	}(data)
 
-	go func(data *summariesData) {
+	go func(data *SummariesData) {
 		defer wg.Done()
 		for i, e := range s.Languages {
 			data.Languages[i] = convertEntry(e, s.TotalTimeBy(models.SummaryLanguage))
@@ -133,14 +133,14 @@ func newDataFrom(s *models.Summary) *summariesData {
 		}
 	}(data)
 
-	go func(data *summariesData) {
+	go func(data *SummariesData) {
 		defer wg.Done()
 		for i, e := range s.OperatingSystems {
 			data.OperatingSystems[i] = convertEntry(e, s.TotalTimeBy(models.SummaryOS))
 		}
 	}(data)
 
-	go func(data *summariesData) {
+	go func(data *SummariesData) {
 		defer wg.Done()
 		for i, e := range s.Machines {
 			data.Machines[i] = convertEntry(e, s.TotalTimeBy(models.SummaryMachine))
@@ -151,7 +151,7 @@ func newDataFrom(s *models.Summary) *summariesData {
 	return data
 }
 
-func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *summariesEntry {
+func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *SummariesEntry {
 	// this is a workaround, since currently, the total time of a summary item is mistakenly represented in seconds
 	// TODO: fix some day, while migrating persisted summary items
 	total := e.Total * time.Second
@@ -163,7 +163,7 @@ func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *summariesEn
 		percentage = 0
 	}
 
-	return &summariesEntry{
+	return &SummariesEntry{
 		Digital:      fmt.Sprintf("%d:%d:%d", hrs, mins, secs),
 		Hours:        hrs,
 		Minutes:      mins,

models/compat/wakatime/v1/user_agent.go

@@ -0,0 +1,12 @@
+package v1
+
+type UserAgentsViewModel struct {
+	Data []*UserAgentEntry `json:"data"`
+}
+
+type UserAgentEntry struct {
+	Id     string
+	Editor string
+	Os     string
+	Value  string
+}

models/heartbeat.go

@@ -19,11 +19,12 @@ type Heartbeat struct {
 	Branch          string         `json:"branch"`
 	Language        string         `json:"language" gorm:"index:idx_language"`
 	IsWrite         bool           `json:"is_write"`
-	Editor          string         `json:"editor"`
-	OperatingSystem string         `json:"operating_system"`
-	Machine         string         `json:"machine"`
+	Editor          string         `json:"editor" hash:"ignore"`           // ignored because editor might be parsed differently by wakatime
+	OperatingSystem string         `json:"operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
+	Machine         string         `json:"machine" hash:"ignore"`          // ignored because wakatime api doesn't return machines currently
 	Time            CustomTime     `json:"time" gorm:"type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time,idx_time_user"`
 	Hash            string         `json:"-" gorm:"type:varchar(17); uniqueIndex"`
+	Origin          string         `json:"-" hash:"ignore"`
 	languageRegex   *regexp.Regexp `hash:"ignore"`
 }
 

repositories/heartbeart.go

@@ -26,6 +26,17 @@ func (r *HeartbeatRepository) InsertBatch(heartbeats []*models.Heartbeat) error
 	return nil
 }
 
+func (r *HeartbeatRepository) CountByUser(user *models.User) (int64, error) {
+	var count int64
+	if err := r.db.
+		Model(&models.Heartbeat{}).
+		Where(&models.Heartbeat{UserID: user.ID}).
+		Count(&count).Error; err != nil {
+		return 0, err
+	}
+	return count, nil
+}
+
 func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
 	var heartbeats []*models.Heartbeat
 	if err := r.db.

repositories/repositories.go

@@ -17,6 +17,7 @@ type IAliasRepository interface {
 
 type IHeartbeatRepository interface {
 	InsertBatch([]*models.Heartbeat) error
+	CountByUser(*models.User) (int64, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	DeleteBefore(time.Time) error

routes/settings.go

@@ -11,6 +11,7 @@ import (
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/models/view"
 	"github.com/muety/wakapi/services"
+	"github.com/muety/wakapi/services/imports"
 	"github.com/muety/wakapi/utils"
 	"net/http"
 	"strconv"
@@ -21,15 +22,25 @@ type SettingsHandler struct {
 	config              *conf.Config
 	userSrvc            services.IUserService
 	summarySrvc         services.ISummaryService
+	heartbeatSrvc       services.IHeartbeatService
 	aliasSrvc           services.IAliasService
 	aggregationSrvc     services.IAggregationService
 	languageMappingSrvc services.ILanguageMappingService
+	keyValueSrvc        services.IKeyValueService
 	httpClient          *http.Client
 }
 
 var credentialsDecoder = schema.NewDecoder()
 
-func NewSettingsHandler(userService services.IUserService, summaryService services.ISummaryService, aliasService services.IAliasService, aggregationService services.IAggregationService, languageMappingService services.ILanguageMappingService) *SettingsHandler {
+func NewSettingsHandler(
+	userService services.IUserService,
+	heartbeatService services.IHeartbeatService,
+	summaryService services.ISummaryService,
+	aliasService services.IAliasService,
+	aggregationService services.IAggregationService,
+	languageMappingService services.ILanguageMappingService,
+	keyValueService services.IKeyValueService,
+) *SettingsHandler {
 	return &SettingsHandler{
 		config:              conf.Get(),
 		summarySrvc:         summaryService,
@@ -37,6 +48,8 @@ func NewSettingsHandler(userService services.IUserService, summaryService servic
 		aggregationSrvc:     aggregationService,
 		languageMappingSrvc: languageMappingService,
 		userSrvc:            userService,
+		heartbeatSrvc:       heartbeatService,
+		keyValueSrvc:        keyValueService,
 		httpClient:          &http.Client{Timeout: 10 * time.Second},
 	}
 }
@@ -88,10 +101,12 @@ func (h *SettingsHandler) PostIndex(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if errorMsg != "" {
+		w.WriteHeader(status)
 		templates[conf.SettingsTemplate].Execute(w, h.buildViewModel(r).WithError(errorMsg))
 		return
 	}
 	if successMsg != "" {
+		w.WriteHeader(status)
 		templates[conf.SettingsTemplate].Execute(w, h.buildViewModel(r).WithSuccess(successMsg))
 		return
 	}
@@ -116,6 +131,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
 		return h.actionToggleBadges
 	case "toggle_wakatime":
 		return h.actionSetWakatimeApiKey
+	case "import_wakatime":
+		return h.actionImportWaktime
 	case "regenerate_summaries":
 		return h.actionRegenerateSummaries
 	case "delete_account":
@@ -315,6 +332,66 @@ func (h *SettingsHandler) actionSetWakatimeApiKey(w http.ResponseWriter, r *http
 	return http.StatusOK, "Wakatime API Key updated successfully", ""
 }
 
+func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Request) (int, string, string) {
+	if h.config.IsDev() {
+		loadTemplates()
+	}
+
+	user := r.Context().Value(models.UserKey).(*models.User)
+	if user.WakatimeApiKey == "" {
+		return http.StatusForbidden, "", "not connected to wakatime"
+	}
+
+	kvKey := fmt.Sprintf("%s_%s", conf.KeyLastImportImport, user.ID)
+
+	if !h.config.IsDev() {
+		lastImportKv := h.keyValueSrvc.MustGetString(kvKey)
+		lastImport, _ := time.Parse(time.RFC822, lastImportKv.Value)
+		if time.Now().Sub(lastImport) < time.Duration(h.config.App.ImportBackoffMin)*time.Minute {
+			return http.StatusTooManyRequests,
+				"",
+				fmt.Sprintf("Too many data imports. You are only allowed to request an import every %d minutes.", h.config.App.ImportBackoffMin)
+		}
+	}
+
+	go func(user *models.User) {
+		importer := imports.NewWakatimeHeartbeatImporter(user.WakatimeApiKey)
+
+		countBefore, err := h.heartbeatSrvc.CountByUser(user)
+		if err != nil {
+			println(err)
+		}
+
+		count := 0
+		batch := make([]*models.Heartbeat, 0)
+
+		for hb := range importer.Import(user) {
+			count++
+			batch = append(batch, hb)
+
+			if len(batch) == h.config.App.ImportBatchSize {
+				if err := h.heartbeatSrvc.InsertBatch(batch); err != nil {
+					logbuch.Warn("failed to insert imported heartbeat, already existing? – %v", err)
+				}
+
+				batch = make([]*models.Heartbeat, 0)
+			}
+		}
+
+		countAfter, _ := h.heartbeatSrvc.CountByUser(user)
+		logbuch.Info("downloaded %d heartbeats for user '%s' (%d actually imported)", count, user.ID, countAfter-countBefore)
+
+		h.regenerateSummaries(user)
+	}(user)
+
+	h.keyValueSrvc.PutString(&models.KeyStringValue{
+		Key:   kvKey,
+		Value: time.Now().Format(time.RFC822),
+	})
+
+	return http.StatusAccepted, "Import started. This may take a few minutes.", ""
+}
+
 func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *http.Request) (int, string, string) {
 	if h.config.IsDev() {
 		loadTemplates()
@@ -322,16 +399,8 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
 
 	user := r.Context().Value(models.UserKey).(*models.User)
 
-	logbuch.Info("clearing summaries for user '%s'", user.ID)
-	if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
-		logbuch.Error("failed to clear summaries: %v", err)
-		return http.StatusInternalServerError, "", "failed to delete old summaries"
-	}
-
-	if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
-		logbuch.Error("failed to regenerate summaries: %v", err)
-		return http.StatusInternalServerError, "", "failed to generate aggregations"
-
+	if err := h.regenerateSummaries(user); err != nil {
+		return http.StatusInternalServerError, "", "failed to regenerate summaries"
 	}
 
 	return http.StatusOK, "summaries are being regenerated – this may take a few seconds", ""
@@ -368,7 +437,7 @@ func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
 
 	request, err := http.NewRequest(
 		http.MethodGet,
-		conf.WakatimeApiUrl+conf.WakatimeApiUserEndpoint,
+		conf.WakatimeApiUrl+conf.WakatimeApiUserUrl,
 		nil,
 	)
 	if err != nil {
@@ -385,6 +454,21 @@ func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
 	return true
 }
 
+func (h *SettingsHandler) regenerateSummaries(user *models.User) error {
+	logbuch.Info("clearing summaries for user '%s'", user.ID)
+	if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+		logbuch.Error("failed to clear summaries: %v", err)
+		return err
+	}
+
+	if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
+		logbuch.Error("failed to regenerate summaries: %v", err)
+		return err
+	}
+
+	return nil
+}
+
 func (h *SettingsHandler) buildViewModel(r *http.Request) *view.SettingsViewModel {
 	user := r.Context().Value(models.UserKey).(*models.User)
 	mappings, _ := h.languageMappingSrvc.GetByUser(user.ID)

services/heartbeat.go

@@ -22,10 +22,18 @@ func NewHeartbeatService(heartbeatRepo repositories.IHeartbeatRepository, langua
 	}
 }
 
+func (srv *HeartbeatService) Insert(heartbeat *models.Heartbeat) error {
+	return srv.repository.InsertBatch([]*models.Heartbeat{heartbeat})
+}
+
 func (srv *HeartbeatService) InsertBatch(heartbeats []*models.Heartbeat) error {
 	return srv.repository.InsertBatch(heartbeats)
 }
 
+func (srv *HeartbeatService) CountByUser(user *models.User) (int64, error) {
+	return srv.repository.CountByUser(user)
+}
+
 func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
 	heartbeats, err := srv.repository.GetAllWithin(from, to, user)
 	if err != nil {

services/imports/importers.go

@@ -0,0 +1,7 @@
+package imports
+
+import "github.com/muety/wakapi/models"
+
+type HeartbeatImporter interface {
+	Import(*models.User) <-chan *models.Heartbeat
+}

services/imports/wakatime.go

@@ -0,0 +1,229 @@
+package imports
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"github.com/muety/wakapi/models"
+	wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
+	"github.com/muety/wakapi/utils"
+	"go.uber.org/atomic"
+	"golang.org/x/sync/semaphore"
+	"net/http"
+	"time"
+)
+
+const (
+	maxWorkers = 6
+)
+
+type WakatimeHeartbeatImporter struct {
+	ApiKey string
+}
+
+func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
+	return &WakatimeHeartbeatImporter{
+		ApiKey: apiKey,
+	}
+}
+
+func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Heartbeat {
+	out := make(chan *models.Heartbeat)
+
+	go func(user *models.User, out chan *models.Heartbeat) {
+		startDate, endDate, err := w.fetchRange()
+		if err != nil {
+			logbuch.Error("failed to fetch date range while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
+			return
+		}
+
+		userAgents, err := w.fetchUserAgents()
+		if err != nil {
+			logbuch.Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
+			return
+		}
+
+		days := generateDays(startDate, endDate)
+
+		c := atomic.NewUint32(uint32(len(days)))
+		ctx := context.TODO()
+		sem := semaphore.NewWeighted(maxWorkers)
+
+		for _, d := range days {
+			if err := sem.Acquire(ctx, 1); err != nil {
+				logbuch.Error("failed to acquire semaphore – %v", err)
+				break
+			}
+
+			go func(day time.Time) {
+				defer sem.Release(1)
+
+				d := day.Format("2006-01-02")
+				heartbeats, err := w.fetchHeartbeats(d)
+				if err != nil {
+					logbuch.Error("failed to fetch heartbeats for day '%s' and user '%s' – &v", day, user.ID, err)
+				}
+
+				for _, h := range heartbeats {
+					out <- mapHeartbeat(h, userAgents, user)
+				}
+
+				if c.Dec() == 0 {
+					close(out)
+				}
+			}(d)
+		}
+	}(user, out)
+
+	return out
+}
+
+// https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
+func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiHeartbeatsUrl, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	q := req.URL.Query()
+	q.Add("date", day)
+	req.URL.RawQuery = q.Encode()
+
+	res, err := httpClient.Do(w.withHeaders(req))
+	if err != nil {
+		return nil, err
+	}
+
+	var heartbeatsData wakatime.HeartbeatsViewModel
+	if err := json.NewDecoder(res.Body).Decode(&heartbeatsData); err != nil {
+		return nil, err
+	}
+
+	return heartbeatsData.Data, nil
+}
+
+// https://wakatime.com/api/v1/users/current/all_time_since_today
+func (w *WakatimeHeartbeatImporter) fetchRange() (time.Time, time.Time, error) {
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	notime := time.Time{}
+
+	req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiAllTimeUrl, nil)
+	if err != nil {
+		return notime, notime, err
+	}
+
+	res, err := httpClient.Do(w.withHeaders(req))
+	if err != nil {
+		return notime, notime, err
+	}
+
+	var allTimeData map[string]interface{}
+	if err := json.NewDecoder(res.Body).Decode(&allTimeData); err != nil {
+		return notime, notime, err
+	}
+
+	data := allTimeData["data"].(map[string]interface{})
+	if data == nil {
+		return notime, notime, errors.New("invalid response")
+	}
+
+	dataRange := data["range"].(map[string]interface{})
+	if dataRange == nil {
+		return notime, notime, errors.New("invalid response")
+	}
+
+	startDate, err := time.Parse("2006-01-02", dataRange["start_date"].(string))
+	if err != nil {
+		return notime, notime, err
+	}
+
+	endDate, err := time.Parse("2006-01-02", dataRange["end_date"].(string))
+	if err != nil {
+		return notime, notime, err
+	}
+
+	return startDate, endDate, nil
+}
+
+// https://wakatime.com/api/v1/users/current/user_agents
+func (w *WakatimeHeartbeatImporter) fetchUserAgents() (map[string]*wakatime.UserAgentEntry, error) {
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+
+	req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiUserAgentsUrl, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	res, err := httpClient.Do(w.withHeaders(req))
+	if err != nil {
+		return nil, err
+	}
+
+	var userAgentsData wakatime.UserAgentsViewModel
+	if err := json.NewDecoder(res.Body).Decode(&userAgentsData); err != nil {
+		return nil, err
+	}
+
+	userAgents := make(map[string]*wakatime.UserAgentEntry)
+	for _, ua := range userAgentsData.Data {
+		userAgents[ua.Id] = ua
+	}
+
+	return userAgents, nil
+}
+
+func (w *WakatimeHeartbeatImporter) withHeaders(req *http.Request) *http.Request {
+	req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(w.ApiKey))))
+	return req
+}
+
+func mapHeartbeat(
+	entry *wakatime.HeartbeatEntry,
+	userAgents map[string]*wakatime.UserAgentEntry,
+	user *models.User,
+) *models.Heartbeat {
+	ua := userAgents[entry.UserAgentId]
+	if ua == nil {
+		ua = &wakatime.UserAgentEntry{
+			Editor: "unknown",
+			Os:     "unknown",
+		}
+	}
+
+	return (&models.Heartbeat{
+		User:            user,
+		UserID:          user.ID,
+		Entity:          entry.Entity,
+		Type:            entry.Type,
+		Category:        entry.Category,
+		Project:         entry.Project,
+		Branch:          entry.Branch,
+		Language:        entry.Language,
+		IsWrite:         entry.IsWrite,
+		Editor:          ua.Editor,
+		OperatingSystem: ua.Os,
+		Machine:         entry.MachineNameId, // TODO
+		Time:            entry.Time,
+		Origin:          fmt.Sprintf("wt@%s", entry.Id),
+	}).Hashed()
+}
+
+func generateDays(from, to time.Time) []time.Time {
+	days := make([]time.Time, 0)
+
+	from = utils.StartOfDay(from)
+	to = utils.StartOfDay(to.Add(24 * time.Hour))
+
+	for d := from; d.Before(to); d = d.Add(24 * time.Hour) {
+		days = append(days, d)
+	}
+
+	return days
+}

services/key_value.go

@@ -22,6 +22,17 @@ func (srv *KeyValueService) GetString(key string) (*models.KeyStringValue, error
 	return srv.repository.GetString(key)
 }
 
+func (srv *KeyValueService) MustGetString(key string) *models.KeyStringValue {
+	kv, err := srv.repository.GetString(key)
+	if err != nil {
+		return &models.KeyStringValue{
+			Key:   key,
+			Value: "",
+		}
+	}
+	return kv
+}
+
 func (srv *KeyValueService) PutString(kv *models.KeyStringValue) error {
 	return srv.repository.PutString(kv)
 }

services/services.go

@@ -26,7 +26,9 @@ type IAliasService interface {
 }
 
 type IHeartbeatService interface {
+	Insert(*models.Heartbeat) error
 	InsertBatch([]*models.Heartbeat) error
+	CountByUser(*models.User) (int64, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	DeleteBefore(time.Time) error
@@ -34,6 +36,7 @@ type IHeartbeatService interface {
 
 type IKeyValueService interface {
 	GetString(string) (*models.KeyStringValue, error)
+	MustGetString(string) *models.KeyStringValue
 	PutString(*models.KeyStringValue) error
 	DeleteString(string) error
 }

version.txt

@@ -1 +1 @@
-1.22.6
+1.23.0

views/settings.tpl.html

@@ -356,17 +356,32 @@
                             </button>
                             {{ else }}
                             <button type="submit"
-                                    class="py-1 px-3 rounded bg-red-500 hover:bg-red-600 text-white text-sm">Disconnect
+                                    class="py-1 px-3 rounded bg-red-500 hover:bg-red-600 text-white text-sm"
+                                    style="width: 130px">Disconnect
                             </button>
                             {{ end }}
                         </div>
                     </div>
+
+                    {{ if .User.WakatimeApiKey }}
+                    <div class="flex justify-end">
+                        <button id="btn-import-wakatime" type="button" style="width: 130px"
+                                class="py-1 px-3 my-3 rounded bg-green-700 hover:bg-green-800 text-white text-sm">
+                            ⤵ Import Data
+                        </button>
+                    </div>
+                    {{ end }}
+                </form>
+
+                <form action="" method="post" id="form-import-wakatime" class="mt-6">
+                    <input type="hidden" name="action" value="import_wakatime">
                 </form>
 
                 <p class="mt-6">
                     <span class="font-semibold">👉 Please note:</span>
                     <span>When enabling this feature, the operators of this server will, in theory (!), have unlimited access to your data stored in WakaTime. If you are concerned about your privacy, please do not enable this integration or wait for OAuth 2 authentication (<a
-                            class="underline" target="_blank" href="https://github.com/muety/wakapi/issues/94" rel="noopener noreferrer">#94</a>) to be implemented.</span>
+                            class="underline" target="_blank" href="https://github.com/muety/wakapi/issues/94"
+                            rel="noopener noreferrer">#94</a>) to be implemented.</span>
                 </p>
             </div>
         </div>
@@ -455,6 +470,12 @@
             formDelete.submit()
         }
     })
+
+    const btnImportWakatime = document.querySelector('#btn-import-wakatime')
+    const formImportWakatime = document.querySelector('#form-import-wakatime')
+    btnImportWakatime.addEventListener('click', () => {
+        formImportWakatime.submit()
+    })
 </script>
 
 {{ template "footer.tpl.html" . }}