mirror of https://github.com/muety/wakapi.git
chore: optimize import date range
This commit is contained in:
parent
da3c80362c
commit
161e375f74
|
@ -68,7 +68,7 @@ I'd love to get some community feedback from active Wakapi users. If you want, p
|
|||
Plans for the near future mainly include, besides usual improvements and bug fixes, a UI redesign as well as additional types of charts and statistics (see [#101](https://github.com/muety/wakapi/issues/101), [#80](https://github.com/muety/wakapi/issues/80), [#76](https://github.com/muety/wakapi/issues/76), [#12](https://github.com/muety/wakapi/issues/12)). If you have feature requests or any kind of improvement proposals feel free to open an issue or share them in our [user survey](https://github.com/muety/wakapi/issues/82).
|
||||
|
||||
## ⌨️ How to use?
|
||||
There are different options for how to use Wakapi, ranging from out hosted cloud service to self-hosting it. Regardless of which option choose, you will always have to do the [client setup](#-client-setup) in addition.
|
||||
There are different options for how to use Wakapi, ranging from our hosted cloud service to self-hosting it. Regardless of which option choose, you will always have to do the [client setup](#-client-setup) in addition.
|
||||
|
||||
### ☁️ Option 1: Use [wakapi.dev](https://wakapi.dev)
|
||||
If you want to you out free, hosted cloud service, all you need to do is create an account and the set up your client-side tooling (see below).
|
||||
|
|
|
@ -25,6 +25,7 @@ type Heartbeat struct {
|
|||
Time CustomTime `json:"time" gorm:"type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time,idx_time_user"`
|
||||
Hash string `json:"-" gorm:"type:varchar(17); uniqueIndex"`
|
||||
Origin string `json:"-" hash:"ignore"`
|
||||
OriginId string `json:"-" hash:"ignore"`
|
||||
languageRegex *regexp.Regexp `hash:"ignore"`
|
||||
}
|
||||
|
||||
|
|
|
@ -37,6 +37,21 @@ func (r *HeartbeatRepository) CountByUser(user *models.User) (int64, error) {
|
|||
return count, nil
|
||||
}
|
||||
|
||||
func (r *HeartbeatRepository) GetLatestByOriginAndUser(origin string, user *models.User) (*models.Heartbeat, error) {
|
||||
var heartbeat models.Heartbeat
|
||||
if err := r.db.
|
||||
Model(&models.Heartbeat{}).
|
||||
Where(&models.Heartbeat{
|
||||
UserID: user.ID,
|
||||
Origin: origin,
|
||||
}).
|
||||
Order("time desc").
|
||||
First(&heartbeat).Error; err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &heartbeat, nil
|
||||
}
|
||||
|
||||
func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User) ([]*models.Heartbeat, error) {
|
||||
var heartbeats []*models.Heartbeat
|
||||
if err := r.db.
|
||||
|
|
|
@ -20,6 +20,7 @@ type IHeartbeatRepository interface {
|
|||
CountByUser(*models.User) (int64, error)
|
||||
GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
|
||||
GetFirstByUsers() ([]*models.TimeByUser, error)
|
||||
GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
|
||||
DeleteBefore(time.Time) error
|
||||
}
|
||||
|
||||
|
|
|
@ -362,10 +362,18 @@ func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Req
|
|||
println(err)
|
||||
}
|
||||
|
||||
var stream <-chan *models.Heartbeat
|
||||
if latest, err := h.heartbeatSrvc.GetLatestByOriginAndUser(imports.OriginWakatime, user); latest == nil || err != nil {
|
||||
stream = importer.ImportAll(user)
|
||||
} else {
|
||||
// if an import has happened before, only import heartbeats newer than the latest of the last import
|
||||
stream = importer.Import(user, latest.Time.T(), time.Now())
|
||||
}
|
||||
|
||||
count := 0
|
||||
batch := make([]*models.Heartbeat, 0)
|
||||
|
||||
for hb := range importer.Import(user) {
|
||||
for hb := range stream {
|
||||
count++
|
||||
batch = append(batch, hb)
|
||||
|
||||
|
@ -389,7 +397,7 @@ func (h *SettingsHandler) actionImportWaktime(w http.ResponseWriter, r *http.Req
|
|||
Value: time.Now().Format(time.RFC822),
|
||||
})
|
||||
|
||||
return http.StatusAccepted, "Import started. This may take a few minutes.", ""
|
||||
return http.StatusAccepted, "ImportAll started. This may take a few minutes.", ""
|
||||
}
|
||||
|
||||
func (h *SettingsHandler) actionRegenerateSummaries(w http.ResponseWriter, r *http.Request) (int, string, string) {
|
||||
|
|
|
@ -42,6 +42,10 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
|
|||
return srv.augmented(heartbeats, user.ID)
|
||||
}
|
||||
|
||||
func (srv *HeartbeatService) GetLatestByOriginAndUser(origin string, user *models.User) (*models.Heartbeat, error) {
|
||||
return srv.repository.GetLatestByOriginAndUser(origin, user)
|
||||
}
|
||||
|
||||
func (srv *HeartbeatService) GetFirstByUsers() ([]*models.TimeByUser, error) {
|
||||
return srv.repository.GetFirstByUsers()
|
||||
}
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
package imports
|
||||
|
||||
import "github.com/muety/wakapi/models"
|
||||
import (
|
||||
"github.com/muety/wakapi/models"
|
||||
"time"
|
||||
)
|
||||
|
||||
type HeartbeatImporter interface {
|
||||
Import(*models.User) <-chan *models.Heartbeat
|
||||
Import(*models.User, time.Time, time.Time) <-chan *models.Heartbeat
|
||||
ImportAll(*models.User) <-chan *models.Heartbeat
|
||||
}
|
||||
|
|
|
@ -17,9 +17,8 @@ import (
|
|||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
maxWorkers = 6
|
||||
)
|
||||
const OriginWakatime = "wakatime"
|
||||
const maxWorkers = 6
|
||||
|
||||
type WakatimeHeartbeatImporter struct {
|
||||
ApiKey string
|
||||
|
@ -31,7 +30,7 @@ func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
|
|||
}
|
||||
}
|
||||
|
||||
func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Heartbeat {
|
||||
func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time, maxTo time.Time) <-chan *models.Heartbeat {
|
||||
out := make(chan *models.Heartbeat)
|
||||
|
||||
go func(user *models.User, out chan *models.Heartbeat) {
|
||||
|
@ -41,6 +40,13 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Hea
|
|||
return
|
||||
}
|
||||
|
||||
if startDate.Before(minFrom) {
|
||||
startDate = minFrom
|
||||
}
|
||||
if endDate.After(maxTo) {
|
||||
endDate = maxTo
|
||||
}
|
||||
|
||||
userAgents, err := w.fetchUserAgents()
|
||||
if err != nil {
|
||||
logbuch.Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' – %v", user.ID, err)
|
||||
|
@ -82,6 +88,10 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User) <-chan *models.Hea
|
|||
return out
|
||||
}
|
||||
|
||||
func (w *WakatimeHeartbeatImporter) ImportAll(user *models.User) <-chan *models.Heartbeat {
|
||||
return w.Import(user, time.Time{}, time.Now())
|
||||
}
|
||||
|
||||
// https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
|
||||
func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
|
||||
httpClient := &http.Client{Timeout: 10 * time.Second}
|
||||
|
@ -211,7 +221,8 @@ func mapHeartbeat(
|
|||
OperatingSystem: ua.Os,
|
||||
Machine: entry.MachineNameId, // TODO
|
||||
Time: entry.Time,
|
||||
Origin: fmt.Sprintf("wt@%s", entry.Id),
|
||||
Origin: OriginWakatime,
|
||||
OriginId: entry.Id,
|
||||
}).Hashed()
|
||||
}
|
||||
|
||||
|
|
|
@ -31,6 +31,7 @@ type IHeartbeatService interface {
|
|||
CountByUser(*models.User) (int64, error)
|
||||
GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
|
||||
GetFirstByUsers() ([]*models.TimeByUser, error)
|
||||
GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)
|
||||
DeleteBefore(time.Time) error
|
||||
}
|
||||
|
||||
|
|
|
@ -474,7 +474,9 @@
|
|||
const btnImportWakatime = document.querySelector('#btn-import-wakatime')
|
||||
const formImportWakatime = document.querySelector('#form-import-wakatime')
|
||||
btnImportWakatime.addEventListener('click', () => {
|
||||
formImportWakatime.submit()
|
||||
if (confirm('Are you sure? The import can not be undone.')) {
|
||||
formImportWakatime.submit()
|
||||
}
|
||||
})
|
||||
</script>
|
||||
|
||||
|
|
Loading…
Reference in New Issue