diff --git a/config/config.go b/config/config.go
index daeac5f..05bcda5 100644
--- a/config/config.go
+++ b/config/config.go
@@ -29,22 +29,28 @@ const (
KeyLatestTotalTime = "latest_total_time"
KeyLatestTotalUsers = "latest_total_users"
+ KeyLastImportImport = "last_import"
)
const (
- WakatimeApiUrl = "https://wakatime.com/api/v1"
- WakatimeApiHeartbeatsEndpoint = "/users/current/heartbeats.bulk"
- WakatimeApiUserEndpoint = "/users/current"
+ WakatimeApiUrl = "https://wakatime.com/api/v1"
+ WakatimeApiUserUrl = "/users/current"
+ WakatimeApiAllTimeUrl = "/users/current/all_time_since_today"
+ WakatimeApiHeartbeatsUrl = "/users/current/heartbeats"
+ WakatimeApiHeartbeatsBulkUrl = "/users/current/heartbeats.bulk"
+ WakatimeApiUserAgentsUrl = "/users/current/user_agents"
)
var cfg *Config
var cFlag = flag.String("config", defaultConfigPath, "config file location")
type appConfig struct {
- AggregationTime string `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"`
- CountingTime string `yaml:"counting_time" default:"05:15" env:"WAKAPI_COUNTING_TIME"`
- CustomLanguages map[string]string `yaml:"custom_languages"`
- Colors map[string]map[string]string `yaml:"-"`
+ AggregationTime string `yaml:"aggregation_time" default:"02:15" env:"WAKAPI_AGGREGATION_TIME"`
+ CountingTime string `yaml:"counting_time" default:"05:15" env:"WAKAPI_COUNTING_TIME"`
+ ImportBackoffMin int `yaml:"import_backoff_min" default:"5" env:"WAKAPI_IMPORT_BACKOFF_MIN"`
+ ImportBatchSize int `yaml:"import_batch_size" default:"25" env:"WAKAPI_IMPORT_BATCH_SIZE"`
+ CustomLanguages map[string]string `yaml:"custom_languages"`
+ Colors map[string]map[string]string `yaml:"-"`
}
type securityConfig struct {
diff --git a/go.mod b/go.mod
index d427bb1..3c617be 100644
--- a/go.mod
+++ b/go.mod
@@ -21,6 +21,7 @@ require (
github.com/stretchr/testify v1.6.1
go.uber.org/atomic v1.6.0
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
+ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect
gopkg.in/yaml.v2 v2.2.8 // indirect
gorm.io/driver/mysql v1.0.3
diff --git a/main.go b/main.go
index 2bf1826..0d5ce1b 100644
--- a/main.go
+++ b/main.go
@@ -136,7 +136,7 @@ func main() {
// MVC Handlers
summaryHandler := routes.NewSummaryHandler(summaryService, userService)
- settingsHandler := routes.NewSettingsHandler(userService, summaryService, aliasService, aggregationService, languageMappingService)
+ settingsHandler := routes.NewSettingsHandler(userService, heartbeatService, summaryService, aliasService, aggregationService, languageMappingService, keyValueService)
homeHandler := routes.NewHomeHandler(keyValueService)
loginHandler := routes.NewLoginHandler(userService)
imprintHandler := routes.NewImprintHandler(keyValueService)
diff --git a/middlewares/custom/wakatime.go b/middlewares/custom/wakatime.go
index eaa669d..c7ab5df 100644
--- a/middlewares/custom/wakatime.go
+++ b/middlewares/custom/wakatime.go
@@ -63,7 +63,7 @@ func (m *WakatimeRelayMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Reque
go m.send(
http.MethodPost,
- config.WakatimeApiUrl+config.WakatimeApiHeartbeatsEndpoint,
+ config.WakatimeApiUrl+config.WakatimeApiHeartbeatsBulkUrl,
bytes.NewReader(body),
headers,
)
diff --git a/models/compat/wakatime/v1/all_time.go b/models/compat/wakatime/v1/all_time.go
index 3d87a32..b2444c1 100644
--- a/models/compat/wakatime/v1/all_time.go
+++ b/models/compat/wakatime/v1/all_time.go
@@ -9,10 +9,10 @@ import (
// https://wakatime.com/developers#all_time_since_today
type AllTimeViewModel struct {
- Data *allTimeData `json:"data"`
+ Data *AllTimeData `json:"data"`
}
-type allTimeData struct {
+type AllTimeData struct {
TotalSeconds float32 `json:"total_seconds"` // total number of seconds logged since account created
Text string `json:"text"` // total time logged since account created as human readable string>
IsUpToDate bool `json:"is_up_to_date"` // true if the stats are up to date; when false, a 202 response code is returned and stats will be refreshed soon>
@@ -27,7 +27,7 @@ func NewAllTimeFrom(summary *models.Summary, filters *models.Filters) *AllTimeVi
}
return &AllTimeViewModel{
- Data: &allTimeData{
+ Data: &AllTimeData{
TotalSeconds: float32(total.Seconds()),
Text: utils.FmtWakatimeDuration(total),
IsUpToDate: true,
diff --git a/models/compat/wakatime/v1/heartbeat.go b/models/compat/wakatime/v1/heartbeat.go
new file mode 100644
index 0000000..8461db4
--- /dev/null
+++ b/models/compat/wakatime/v1/heartbeat.go
@@ -0,0 +1,25 @@
+package v1
+
+import "github.com/muety/wakapi/models"
+
+type HeartbeatsViewModel struct {
+ Data []*HeartbeatEntry `json:"data"`
+}
+
+// Incomplete, for now, only the subset of fields is implemented
+// that is actually required for the import
+
+type HeartbeatEntry struct {
+ Id string
+ Branch string
+ Category string
+ Entity string
+ IsWrite bool `json:"is_write"`
+ Language string
+ Project string
+ Time models.CustomTime
+ Type string
+ UserId string `json:"user_id"`
+ MachineNameId string `json:"machine_name_id"`
+ UserAgentId string `json:"user_agent_id"`
+}
diff --git a/models/compat/wakatime/v1/summaries.go b/models/compat/wakatime/v1/summaries.go
index 9383e02..23cf1d3 100644
--- a/models/compat/wakatime/v1/summaries.go
+++ b/models/compat/wakatime/v1/summaries.go
@@ -13,24 +13,24 @@ import (
// https://pastr.de/v/736450
type SummariesViewModel struct {
- Data []*summariesData `json:"data"`
+ Data []*SummariesData `json:"data"`
End time.Time `json:"end"`
Start time.Time `json:"start"`
}
-type summariesData struct {
- Categories []*summariesEntry `json:"categories"`
- Dependencies []*summariesEntry `json:"dependencies"`
- Editors []*summariesEntry `json:"editors"`
- Languages []*summariesEntry `json:"languages"`
- Machines []*summariesEntry `json:"machines"`
- OperatingSystems []*summariesEntry `json:"operating_systems"`
- Projects []*summariesEntry `json:"projects"`
- GrandTotal *summariesGrandTotal `json:"grand_total"`
- Range *summariesRange `json:"range"`
+type SummariesData struct {
+ Categories []*SummariesEntry `json:"categories"`
+ Dependencies []*SummariesEntry `json:"dependencies"`
+ Editors []*SummariesEntry `json:"editors"`
+ Languages []*SummariesEntry `json:"languages"`
+ Machines []*SummariesEntry `json:"machines"`
+ OperatingSystems []*SummariesEntry `json:"operating_systems"`
+ Projects []*SummariesEntry `json:"projects"`
+ GrandTotal *SummariesGrandTotal `json:"grand_total"`
+ Range *SummariesRange `json:"range"`
}
-type summariesEntry struct {
+type SummariesEntry struct {
Digital string `json:"digital"`
Hours int `json:"hours"`
Minutes int `json:"minutes"`
@@ -41,7 +41,7 @@ type summariesEntry struct {
TotalSeconds float64 `json:"total_seconds"`
}
-type summariesGrandTotal struct {
+type SummariesGrandTotal struct {
Digital string `json:"digital"`
Hours int `json:"hours"`
Minutes int `json:"minutes"`
@@ -49,7 +49,7 @@ type summariesGrandTotal struct {
TotalSeconds float64 `json:"total_seconds"`
}
-type summariesRange struct {
+type SummariesRange struct {
Date string `json:"date"`
End time.Time `json:"end"`
Start time.Time `json:"start"`
@@ -58,7 +58,7 @@ type summariesRange struct {
}
func NewSummariesFrom(summaries []*models.Summary, filters *models.Filters) *SummariesViewModel {
- data := make([]*summariesData, len(summaries))
+ data := make([]*SummariesData, len(summaries))
minDate, maxDate := time.Now().Add(1*time.Second), time.Time{}
for i, s := range summaries {
@@ -79,27 +79,27 @@ func NewSummariesFrom(summaries []*models.Summary, filters *models.Filters) *Sum
}
}
-func newDataFrom(s *models.Summary) *summariesData {
+func newDataFrom(s *models.Summary) *SummariesData {
zone, _ := time.Now().Zone()
total := s.TotalTime()
totalHrs, totalMins := int(total.Hours()), int((total - time.Duration(total.Hours())*time.Hour).Minutes())
- data := &summariesData{
- Categories: make([]*summariesEntry, 0),
- Dependencies: make([]*summariesEntry, 0),
- Editors: make([]*summariesEntry, len(s.Editors)),
- Languages: make([]*summariesEntry, len(s.Languages)),
- Machines: make([]*summariesEntry, len(s.Machines)),
- OperatingSystems: make([]*summariesEntry, len(s.OperatingSystems)),
- Projects: make([]*summariesEntry, len(s.Projects)),
- GrandTotal: &summariesGrandTotal{
+ data := &SummariesData{
+ Categories: make([]*SummariesEntry, 0),
+ Dependencies: make([]*SummariesEntry, 0),
+ Editors: make([]*SummariesEntry, len(s.Editors)),
+ Languages: make([]*SummariesEntry, len(s.Languages)),
+ Machines: make([]*SummariesEntry, len(s.Machines)),
+ OperatingSystems: make([]*SummariesEntry, len(s.OperatingSystems)),
+ Projects: make([]*SummariesEntry, len(s.Projects)),
+ GrandTotal: &SummariesGrandTotal{
Digital: fmt.Sprintf("%d:%d", totalHrs, totalMins),
Hours: totalHrs,
Minutes: totalMins,
Text: utils.FmtWakatimeDuration(total),
TotalSeconds: total.Seconds(),
},
- Range: &summariesRange{
+ Range: &SummariesRange{
Date: time.Now().Format(time.RFC3339),
End: s.ToTime.T(),
Start: s.FromTime.T(),
@@ -111,21 +111,21 @@ func newDataFrom(s *models.Summary) *summariesData {
var wg sync.WaitGroup
wg.Add(5)
- go func(data *summariesData) {
+ go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Projects {
data.Projects[i] = convertEntry(e, s.TotalTimeBy(models.SummaryProject))
}
}(data)
- go func(data *summariesData) {
+ go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Editors {
data.Editors[i] = convertEntry(e, s.TotalTimeBy(models.SummaryEditor))
}
}(data)
- go func(data *summariesData) {
+ go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Languages {
data.Languages[i] = convertEntry(e, s.TotalTimeBy(models.SummaryLanguage))
@@ -133,14 +133,14 @@ func newDataFrom(s *models.Summary) *summariesData {
}
}(data)
- go func(data *summariesData) {
+ go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.OperatingSystems {
data.OperatingSystems[i] = convertEntry(e, s.TotalTimeBy(models.SummaryOS))
}
}(data)
- go func(data *summariesData) {
+ go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Machines {
data.Machines[i] = convertEntry(e, s.TotalTimeBy(models.SummaryMachine))
@@ -151,7 +151,7 @@ func newDataFrom(s *models.Summary) *summariesData {
return data
}
-func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *summariesEntry {
+func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *SummariesEntry {
// this is a workaround, since currently, the total time of a summary item is mistakenly represented in seconds
// TODO: fix some day, while migrating persisted summary items
total := e.Total * time.Second
@@ -163,7 +163,7 @@ func convertEntry(e *models.SummaryItem, entityTotal time.Duration) *summariesEn
percentage = 0
}
- return &summariesEntry{
+ return &SummariesEntry{
Digital: fmt.Sprintf("%d:%d:%d", hrs, mins, secs),
Hours: hrs,
Minutes: mins,
diff --git a/models/compat/wakatime/v1/user_agent.go b/models/compat/wakatime/v1/user_agent.go
new file mode 100644
index 0000000..a73a755
--- /dev/null
+++ b/models/compat/wakatime/v1/user_agent.go
@@ -0,0 +1,12 @@
+package v1
+
+type UserAgentsViewModel struct {
+ Data []*UserAgentEntry `json:"data"`
+}
+
+type UserAgentEntry struct {
+ Id string
+ Editor string
+ Os string
+ Value string
+}
diff --git a/models/heartbeat.go b/models/heartbeat.go
index 9412fbd..dc68c7f 100644
--- a/models/heartbeat.go
+++ b/models/heartbeat.go
@@ -19,11 +19,12 @@ type Heartbeat struct {
Branch string `json:"branch"`
Language string `json:"language" gorm:"index:idx_language"`
IsWrite bool `json:"is_write"`
- Editor string `json:"editor"`
- OperatingSystem string `json:"operating_system"`
- Machine string `json:"machine"`
+ Editor string `json:"editor" hash:"ignore"` // ignored because editor might be parsed differently by wakatime
+ OperatingSystem string `json:"operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
+ Machine string `json:"machine" hash:"ignore"` // ignored because wakatime api doesn't return machines currently
Time CustomTime `json:"time" gorm:"type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time,idx_time_user"`
Hash string `json:"-" gorm:"type:varchar(17); uniqueIndex"`
+ Origin string `json:"-" hash:"ignore"`
languageRegex *regexp.Regexp `hash:"ignore"`
}
diff --git a/repositories/heartbeart.go b/repositories/heartbeart.go
index 9de9f67..631642e 100644
--- a/repositories/heartbeart.go
+++ b/repositories/heartbeart.go
@@ -26,6 +26,17 @@ func (r *HeartbeatRepository) InsertBatch(heartbeats []*models.Heartbeat) error
return nil
}
+func (r *HeartbeatRepository) CountByUser(user *models.User) (int64, error) {
+ var count int64
+ if err := r.db.
+ Model(&models.Heartbeat{}).
+ Where(&models.Heartbeat{UserID: user.ID}).
+ Count(&count).Error; err != nil {
+ return 0, err
+ }
+ return count, nil
+}
+
func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
var heartbeats []*models.Heartbeat
if err := r.db.
diff --git a/repositories/repositories.go b/repositories/repositories.go
index 2186d88..588e888 100644
--- a/repositories/repositories.go
+++ b/repositories/repositories.go
@@ -17,6 +17,7 @@ type IAliasRepository interface {
type IHeartbeatRepository interface {
InsertBatch([]*models.Heartbeat) error
+ CountByUser(*models.User) (int64, error)
GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
GetFirstByUsers() ([]*models.TimeByUser, error)
DeleteBefore(time.Time) error
diff --git a/routes/settings.go b/routes/settings.go
index f46f1d0..f8ccfb8 100644
--- a/routes/settings.go
+++ b/routes/settings.go
@@ -11,6 +11,7 @@ import (
"github.com/muety/wakapi/models"
"github.com/muety/wakapi/models/view"
"github.com/muety/wakapi/services"
+ "github.com/muety/wakapi/services/imports"
"github.com/muety/wakapi/utils"
"net/http"
"strconv"
@@ -21,15 +22,25 @@ type SettingsHandler struct {
config *conf.Config
userSrvc services.IUserService
summarySrvc services.ISummaryService
+ heartbeatSrvc services.IHeartbeatService
aliasSrvc services.IAliasService
aggregationSrvc services.IAggregationService
languageMappingSrvc services.ILanguageMappingService
+ keyValueSrvc services.IKeyValueService
httpClient *http.Client
}
var credentialsDecoder = schema.NewDecoder()
-func NewSettingsHandler(userService services.IUserService, summaryService services.ISummaryService, aliasService services.IAliasService, aggregationService services.IAggregationService, languageMappingService services.ILanguageMappingService) *SettingsHandler {
+func NewSettingsHandler(
+ userService services.IUserService,
+ heartbeatService services.IHeartbeatService,
+ summaryService services.ISummaryService,
+ aliasService services.IAliasService,
+ aggregationService services.IAggregationService,
+ languageMappingService services.ILanguageMappingService,
+ keyValueService services.IKeyValueService,
+) *SettingsHandler {
return &SettingsHandler{
config: conf.Get(),
summarySrvc: summaryService,
@@ -37,6 +48,8 @@ func NewSettingsHandler(userService services.IUserService, summaryService servic
aggregationSrvc: aggregationService,
languageMappingSrvc: languageMappingService,
userSrvc: userService,
+ heartbeatSrvc: heartbeatService,
+ keyValueSrvc: keyValueService,
httpClient: &http.Client{Timeout: 10 * time.Second},
}
}
@@ -88,10 +101,12 @@ func (h *SettingsHandler) PostIndex(w http.ResponseWriter, r *http.Request) {
}
if errorMsg != "" {
+ w.WriteHeader(status)
templates[conf.SettingsTemplate].Execute(w, h.buildViewModel(r).WithError(errorMsg))
return
}
if successMsg != "" {
+ w.WriteHeader(status)
templates[conf.SettingsTemplate].Execute(w, h.buildViewModel(r).WithSuccess(successMsg))
return
}
@@ -116,6 +131,8 @@ func (h *SettingsHandler) dispatchAction(action string) action {
return h.actionToggleBadges
case "toggle_wakatime":
return h.actionSetWakatimeApiKey
+ case "import_wakatime":
+ return h.actionImportWaktime
case "regenerate_summaries":
return h.actionRegenerateSummaries
case "delete_account":
@@ -315,6 +332,66 @@ func (h *SettingsHandler) actionSetWakatimeApiKey(w http.ResponseWriter, r *http
return http.StatusOK, "Wakatime API Key updated successfully", ""
}
+func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Request) (int, string, string) {
+ if h.config.IsDev() {
+ loadTemplates()
+ }
+
+ user := r.Context().Value(models.UserKey).(*models.User)
+ if user.WakatimeApiKey == "" {
+ return http.StatusForbidden, "", "not connected to wakatime"
+ }
+
+ kvKey := fmt.Sprintf("%s_%s", conf.KeyLastImportImport, user.ID)
+
+ if !h.config.IsDev() {
+ lastImportKv := h.keyValueSrvc.MustGetString(kvKey)
+ lastImport, _ := time.Parse(time.RFC822, lastImportKv.Value)
+ if time.Now().Sub(lastImport) < time.Duration(h.config.App.ImportBackoffMin)*time.Minute {
+ return http.StatusTooManyRequests,
+ "",
+ fmt.Sprintf("Too many data imports. You are only allowed to request an import every %d minutes.", h.config.App.ImportBackoffMin)
+ }
+ }
+
+ go func(user *models.User) {
+ importer := imports.NewWakatimeHeartbeatImporter(user.WakatimeApiKey)
+
+ countBefore, err := h.heartbeatSrvc.CountByUser(user)
+ if err != nil {
+ println(err)
+ }
+
+ count := 0
+ batch := make([]*models.Heartbeat, 0)
+
+ for hb := range importer.Import(user) {
+ count++
+ batch = append(batch, hb)
+
+ if len(batch) == h.config.App.ImportBatchSize {
+ if err := h.heartbeatSrvc.InsertBatch(batch); err != nil {
+ logbuch.Warn("failed to insert imported heartbeat, already existing? – %v", err)
+ }
+
+ batch = make([]*models.Heartbeat, 0)
+ }
+ }
+
+ countAfter, _ := h.heartbeatSrvc.CountByUser(user)
+ logbuch.Info("downloaded %d heartbeats for user '%s' (%d actually imported)", count, user.ID, countAfter-countBefore)
+
+ h.regenerateSummaries(user)
+ }(user)
+
+ h.keyValueSrvc.PutString(&models.KeyStringValue{
+ Key: kvKey,
+ Value: time.Now().Format(time.RFC822),
+ })
+
+ return http.StatusAccepted, "Import started. This may take a few minutes.", ""
+}
+
func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *http.Request) (int, string, string) {
if h.config.IsDev() {
loadTemplates()
@@ -322,16 +399,8 @@ func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *ht
user := r.Context().Value(models.UserKey).(*models.User)
- logbuch.Info("clearing summaries for user '%s'", user.ID)
- if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
- logbuch.Error("failed to clear summaries: %v", err)
- return http.StatusInternalServerError, "", "failed to delete old summaries"
- }
-
- if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
- logbuch.Error("failed to regenerate summaries: %v", err)
- return http.StatusInternalServerError, "", "failed to generate aggregations"
-
+ if err := h.regenerateSummaries(user); err != nil {
+ return http.StatusInternalServerError, "", "failed to regenerate summaries"
}
return http.StatusOK, "summaries are being regenerated – this may take a few seconds", ""
@@ -368,7 +437,7 @@ func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
request, err := http.NewRequest(
http.MethodGet,
- conf.WakatimeApiUrl+conf.WakatimeApiUserEndpoint,
+ conf.WakatimeApiUrl+conf.WakatimeApiUserUrl,
nil,
)
if err != nil {
@@ -385,6 +454,21 @@ func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
return true
}
+func (h *SettingsHandler) regenerateSummaries(user *models.User) error {
+ logbuch.Info("clearing summaries for user '%s'", user.ID)
+ if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
+ logbuch.Error("failed to clear summaries: %v", err)
+ return err
+ }
+
+ if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
+ logbuch.Error("failed to regenerate summaries: %v", err)
+ return err
+ }
+
+ return nil
+}
+
func (h *SettingsHandler) buildViewModel(r *http.Request) *view.SettingsViewModel {
user := r.Context().Value(models.UserKey).(*models.User)
mappings, _ := h.languageMappingSrvc.GetByUser(user.ID)
diff --git a/services/heartbeat.go b/services/heartbeat.go
index da8dd53..656fe5e 100644
--- a/services/heartbeat.go
+++ b/services/heartbeat.go
@@ -22,10 +22,18 @@ func NewHeartbeatService(heartbeatRepo repositories.IHeartbeatRepository, langua
}
}
+func (srv *HeartbeatService) Insert(heartbeat *models.Heartbeat) error {
+ return srv.repository.InsertBatch([]*models.Heartbeat{heartbeat})
+}
+
func (srv *HeartbeatService) InsertBatch(heartbeats []*models.Heartbeat) error {
return srv.repository.InsertBatch(heartbeats)
}
+func (srv *HeartbeatService) CountByUser(user *models.User) (int64, error) {
+ return srv.repository.CountByUser(user)
+}
+
func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
heartbeats, err := srv.repository.GetAllWithin(from, to, user)
if err != nil {
diff --git a/services/imports/importers.go b/services/imports/importers.go
new file mode 100644
index 0000000..2316461
--- /dev/null
+++ b/services/imports/importers.go
@@ -0,0 +1,7 @@
+package imports
+
+import "github.com/muety/wakapi/models"
+
+type HeartbeatImporter interface {
+ Import(*models.User) <-chan *models.Heartbeat
+}
diff --git a/services/imports/wakatime.go b/services/imports/wakatime.go
new file mode 100644
index 0000000..1f3c7f2
--- /dev/null
+++ b/services/imports/wakatime.go
@@ -0,0 +1,229 @@
+package imports
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "github.com/emvi/logbuch"
+ "github.com/muety/wakapi/config"
+ "github.com/muety/wakapi/models"
+ wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
+ "github.com/muety/wakapi/utils"
+ "go.uber.org/atomic"
+ "golang.org/x/sync/semaphore"
+ "net/http"
+ "time"
+)
+
+const (
+ maxWorkers = 6
+)
+
+type WakatimeHeartbeatImporter struct {
+ ApiKey string
+}
+
+func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
+ return &WakatimeHeartbeatImporter{
+ ApiKey: apiKey,
+ }
+}
+
+func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Heartbeat {
+ out := make(chan *models.Heartbeat)
+
+ go func(user *models.User, out chan *models.Heartbeat) {
+ startDate, endDate, err := w.fetchRange()
+ if err != nil {
+ logbuch.Error("failed to fetch date range while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
+ return
+ }
+
+ userAgents, err := w.fetchUserAgents()
+ if err != nil {
+ logbuch.Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
+ return
+ }
+
+ days := generateDays(startDate, endDate)
+
+ c := atomic.NewUint32(uint32(len(days)))
+ ctx := context.TODO()
+ sem := semaphore.NewWeighted(maxWorkers)
+
+ for _, d := range days {
+ if err := sem.Acquire(ctx, 1); err != nil {
+ logbuch.Error("failed to acquire semaphore – %v", err)
+ break
+ }
+
+ go func(day time.Time) {
+ defer sem.Release(1)
+
+ d := day.Format("2006-01-02")
+ heartbeats, err := w.fetchHeartbeats(d)
+ if err != nil {
+ logbuch.Error("failed to fetch heartbeats for day '%s' and user '%s' – &v", day, user.ID, err)
+ }
+
+ for _, h := range heartbeats {
+ out <- mapHeartbeat(h, userAgents, user)
+ }
+
+ if c.Dec() == 0 {
+ close(out)
+ }
+ }(d)
+ }
+ }(user, out)
+
+ return out
+}
+
+// https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
+func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
+ httpClient := &http.Client{Timeout: 10 * time.Second}
+
+ req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiHeartbeatsUrl, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ q := req.URL.Query()
+ q.Add("date", day)
+ req.URL.RawQuery = q.Encode()
+
+ res, err := httpClient.Do(w.withHeaders(req))
+ if err != nil {
+ return nil, err
+ }
+
+ var heartbeatsData wakatime.HeartbeatsViewModel
+ if err := json.NewDecoder(res.Body).Decode(&heartbeatsData); err != nil {
+ return nil, err
+ }
+
+ return heartbeatsData.Data, nil
+}
+
+// https://wakatime.com/api/v1/users/current/all_time_since_today
+func (w *WakatimeHeartbeatImporter) fetchRange() (time.Time, time.Time, error) {
+ httpClient := &http.Client{Timeout: 10 * time.Second}
+
+ notime := time.Time{}
+
+ req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiAllTimeUrl, nil)
+ if err != nil {
+ return notime, notime, err
+ }
+
+ res, err := httpClient.Do(w.withHeaders(req))
+ if err != nil {
+ return notime, notime, err
+ }
+
+ var allTimeData map[string]interface{}
+ if err := json.NewDecoder(res.Body).Decode(&allTimeData); err != nil {
+ return notime, notime, err
+ }
+
+ data := allTimeData["data"].(map[string]interface{})
+ if data == nil {
+ return notime, notime, errors.New("invalid response")
+ }
+
+ dataRange := data["range"].(map[string]interface{})
+ if dataRange == nil {
+ return notime, notime, errors.New("invalid response")
+ }
+
+ startDate, err := time.Parse("2006-01-02", dataRange["start_date"].(string))
+ if err != nil {
+ return notime, notime, err
+ }
+
+ endDate, err := time.Parse("2006-01-02", dataRange["end_date"].(string))
+ if err != nil {
+ return notime, notime, err
+ }
+
+ return startDate, endDate, nil
+}
+
+// https://wakatime.com/api/v1/users/current/user_agents
+func (w *WakatimeHeartbeatImporter) fetchUserAgents() (map[string]*wakatime.UserAgentEntry, error) {
+ httpClient := &http.Client{Timeout: 10 * time.Second}
+
+ req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiUserAgentsUrl, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ res, err := httpClient.Do(w.withHeaders(req))
+ if err != nil {
+ return nil, err
+ }
+
+ var userAgentsData wakatime.UserAgentsViewModel
+ if err := json.NewDecoder(res.Body).Decode(&userAgentsData); err != nil {
+ return nil, err
+ }
+
+ userAgents := make(map[string]*wakatime.UserAgentEntry)
+ for _, ua := range userAgentsData.Data {
+ userAgents[ua.Id] = ua
+ }
+
+ return userAgents, nil
+}
+
+func (w *WakatimeHeartbeatImporter) withHeaders(req *http.Request) *http.Request {
+ req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(w.ApiKey))))
+ return req
+}
+
+func mapHeartbeat(
+ entry *wakatime.HeartbeatEntry,
+ userAgents map[string]*wakatime.UserAgentEntry,
+ user *models.User,
+) *models.Heartbeat {
+ ua := userAgents[entry.UserAgentId]
+ if ua == nil {
+ ua = &wakatime.UserAgentEntry{
+ Editor: "unknown",
+ Os: "unknown",
+ }
+ }
+
+ return (&models.Heartbeat{
+ User: user,
+ UserID: user.ID,
+ Entity: entry.Entity,
+ Type: entry.Type,
+ Category: entry.Category,
+ Project: entry.Project,
+ Branch: entry.Branch,
+ Language: entry.Language,
+ IsWrite: entry.IsWrite,
+ Editor: ua.Editor,
+ OperatingSystem: ua.Os,
+ Machine: entry.MachineNameId, // TODO
+ Time: entry.Time,
+ Origin: fmt.Sprintf("wt@%s", entry.Id),
+ }).Hashed()
+}
+
+func generateDays(from, to time.Time) []time.Time {
+ days := make([]time.Time, 0)
+
+ from = utils.StartOfDay(from)
+ to = utils.StartOfDay(to.Add(24 * time.Hour))
+
+ for d := from; d.Before(to); d = d.Add(24 * time.Hour) {
+ days = append(days, d)
+ }
+
+ return days
+}
diff --git a/services/key_value.go b/services/key_value.go
index 11c7e67..deef6bb 100644
--- a/services/key_value.go
+++ b/services/key_value.go
@@ -22,6 +22,17 @@ func (srv *KeyValueService) GetString(key string) (*models.KeyStringValue, error
return srv.repository.GetString(key)
}
+func (srv *KeyValueService) MustGetString(key string) *models.KeyStringValue {
+ kv, err := srv.repository.GetString(key)
+ if err != nil {
+ return &models.KeyStringValue{
+ Key: key,
+ Value: "",
+ }
+ }
+ return kv
+}
+
func (srv *KeyValueService) PutString(kv *models.KeyStringValue) error {
return srv.repository.PutString(kv)
}
diff --git a/services/services.go b/services/services.go
index 1b638ea..02d73bc 100644
--- a/services/services.go
+++ b/services/services.go
@@ -26,7 +26,9 @@ type IAliasService interface {
}
type IHeartbeatService interface {
+ Insert(*models.Heartbeat) error
InsertBatch([]*models.Heartbeat) error
+ CountByUser(*models.User) (int64, error)
GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
GetFirstByUsers() ([]*models.TimeByUser, error)
DeleteBefore(time.Time) error
@@ -34,6 +36,7 @@ type IHeartbeatService interface {
type IKeyValueService interface {
GetString(string) (*models.KeyStringValue, error)
+ MustGetString(string) *models.KeyStringValue
PutString(*models.KeyStringValue) error
DeleteString(string) error
}
diff --git a/version.txt b/version.txt
index e81ed8b..bfbadb3 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-1.22.6
\ No newline at end of file
+1.23.0
\ No newline at end of file
diff --git a/views/settings.tpl.html b/views/settings.tpl.html
index 3187569..9085bea 100644
--- a/views/settings.tpl.html
+++ b/views/settings.tpl.html
@@ -356,17 +356,32 @@
{{ else }}
{{ end }}
+
+ {{ if .User.WakatimeApiKey }}
+
+
+
+ {{ end }}
+
+
+
👉 Please note:When enabling this feature, the operators of this server will, in theory (!), have unlimited access to your data stored in WakaTime. If you are concerned about your privacy, please do not enable this integration or wait for OAuth 2 authentication (#94) to be implemented.
+ class="underline" target="_blank" href="https://github.com/muety/wakapi/issues/94"
+ rel="noopener noreferrer">#94) to be implemented.