1
0
mirror of https://github.com/muety/wakapi.git synced 2023-08-10 21:12:56 +03:00
wakapi/services/imports/wakatime.go

303 lines
7.8 KiB
Go
Raw Normal View History

package imports
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/emvi/logbuch"
"github.com/muety/wakapi/config"
"github.com/muety/wakapi/models"
wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
"github.com/muety/wakapi/utils"
"go.uber.org/atomic"
"golang.org/x/sync/semaphore"
"net/http"
"time"
)
2021-02-06 02:31:30 +03:00
const OriginWakatime = "wakatime"
const (
// wakatime api permits a max. rate of 10 req / sec
// https://github.com/wakatime/wakatime/issues/261
// with 5 workers, each sleeping slightly over 1/2 sec after every req., we should stay well below that limit
maxWorkers = 5
throttleDelay = 550 * time.Millisecond
)
type WakatimeHeartbeatImporter struct {
ApiKey string
}
func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
return &WakatimeHeartbeatImporter{
ApiKey: apiKey,
}
}
2021-02-06 02:31:30 +03:00
func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time, maxTo time.Time) <-chan *models.Heartbeat {
out := make(chan *models.Heartbeat)
go func(user *models.User, out chan *models.Heartbeat) {
startDate, endDate, err := w.fetchRange()
if err != nil {
2021-04-16 16:59:39 +03:00
config.Log().Error("failed to fetch date range while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
}
2021-02-06 02:31:30 +03:00
if startDate.Before(minFrom) {
startDate = minFrom
}
if endDate.After(maxTo) {
endDate = maxTo
}
userAgents, err := w.fetchUserAgents()
if err != nil {
2021-04-16 16:59:39 +03:00
config.Log().Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
}
machinesNames, err := w.fetchMachineNames()
if err != nil {
2021-04-16 16:59:39 +03:00
config.Log().Error("failed to fetch machine names while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
}
days := generateDays(startDate, endDate)
c := atomic.NewUint32(uint32(len(days)))
ctx := context.TODO()
sem := semaphore.NewWeighted(maxWorkers)
for _, d := range days {
if err := sem.Acquire(ctx, 1); err != nil {
logbuch.Error("failed to acquire semaphore %v", err)
break
}
go func(day time.Time) {
defer sem.Release(1)
defer time.Sleep(throttleDelay)
2021-02-12 21:25:59 +03:00
d := day.Format(config.SimpleDateFormat)
heartbeats, err := w.fetchHeartbeats(d)
if err != nil {
2021-04-16 16:59:39 +03:00
config.Log().Error("failed to fetch heartbeats for day '%s' and user '%s' &v", d, user.ID, err)
}
for _, h := range heartbeats {
out <- mapHeartbeat(h, userAgents, machinesNames, user)
}
if c.Dec() == 0 {
close(out)
}
}(d)
}
}(user, out)
return out
}
2021-02-06 02:31:30 +03:00
func (w *WakatimeHeartbeatImporter) ImportAll(user *models.User) <-chan *models.Heartbeat {
return w.Import(user, time.Time{}, time.Now())
}
// https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
2021-08-29 11:54:00 +03:00
// https://pastr.de/p/b5p4od5s8w0pfntmwoi117jy
func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiHeartbeatsUrl, nil)
if err != nil {
return nil, err
}
q := req.URL.Query()
q.Add("date", day)
req.URL.RawQuery = q.Encode()
res, err := httpClient.Do(w.withHeaders(req))
if err != nil {
return nil, err
} else if res.StatusCode >= 400 {
return nil, errors.New(fmt.Sprintf("got status %d from wakatime api", res.StatusCode))
}
var heartbeatsData wakatime.HeartbeatsViewModel
if err := json.NewDecoder(res.Body).Decode(&heartbeatsData); err != nil {
return nil, err
}
return heartbeatsData.Data, nil
}
// https://wakatime.com/api/v1/users/current/all_time_since_today
2021-08-29 11:54:00 +03:00
// https://pastr.de/p/w8xb4biv575pu32pox7jj2gr
func (w *WakatimeHeartbeatImporter) fetchRange() (time.Time, time.Time, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
notime := time.Time{}
req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiAllTimeUrl, nil)
if err != nil {
return notime, notime, err
}
res, err := httpClient.Do(w.withHeaders(req))
if err != nil {
return notime, notime, err
}
var allTimeData wakatime.AllTimeViewModel
if err := json.NewDecoder(res.Body).Decode(&allTimeData); err != nil {
return notime, notime, err
}
2021-02-12 21:25:59 +03:00
startDate, err := time.Parse(config.SimpleDateFormat, allTimeData.Data.Range.StartDate)
if err != nil {
return notime, notime, err
}
2021-02-12 21:25:59 +03:00
endDate, err := time.Parse(config.SimpleDateFormat, allTimeData.Data.Range.EndDate)
if err != nil {
return notime, notime, err
}
return startDate, endDate, nil
}
// https://wakatime.com/api/v1/users/current/user_agents
2021-08-29 11:54:00 +03:00
// https://pastr.de/p/05k5do8q108k94lic4lfl3pc
func (w *WakatimeHeartbeatImporter) fetchUserAgents() (map[string]*wakatime.UserAgentEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
userAgents := make(map[string]*wakatime.UserAgentEntry)
2022-01-02 13:06:00 +03:00
for page := 1; ; page++ {
url := fmt.Sprintf("%s%s?page=%d", config.WakatimeApiUrl, config.WakatimeApiUserAgentsUrl, page)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
res, err := httpClient.Do(w.withHeaders(req))
if err != nil {
return nil, err
}
var userAgentsData wakatime.UserAgentsViewModel
if err := json.NewDecoder(res.Body).Decode(&userAgentsData); err != nil {
return nil, err
}
for _, ua := range userAgentsData.Data {
userAgents[ua.Id] = ua
}
2022-01-02 13:06:00 +03:00
if page == userAgentsData.TotalPages {
break
}
}
return userAgents, nil
}
// https://wakatime.com/api/v1/users/current/machine_names
2021-08-29 11:54:00 +03:00
// https://pastr.de/p/v58cv0xrupp3zvyyv8o6973j
func (w *WakatimeHeartbeatImporter) fetchMachineNames() (map[string]*wakatime.MachineEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
machines := make(map[string]*wakatime.MachineEntry)
2022-01-02 13:06:00 +03:00
for page := 1; ; page++ {
url := fmt.Sprintf("%s%s?page=%d", config.WakatimeApiUrl, config.WakatimeApiMachineNamesUrl, page)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
}
res, err := httpClient.Do(w.withHeaders(req))
if err != nil {
return nil, err
}
var machineData wakatime.MachineViewModel
if err := json.NewDecoder(res.Body).Decode(&machineData); err != nil {
2022-01-02 13:06:00 +03:00
return nil, err
}
for _, ma := range machineData.Data {
machines[ma.Id] = ma
}
2022-01-02 13:06:00 +03:00
if page == machineData.TotalPages {
break
}
}
return machines, nil
}
func (w *WakatimeHeartbeatImporter) withHeaders(req *http.Request) *http.Request {
req.Header.Set("Authorization", fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(w.ApiKey))))
return req
}
func mapHeartbeat(
entry *wakatime.HeartbeatEntry,
userAgents map[string]*wakatime.UserAgentEntry,
machineNames map[string]*wakatime.MachineEntry,
user *models.User,
) *models.Heartbeat {
ua := userAgents[entry.UserAgentId]
if ua == nil {
ua = &wakatime.UserAgentEntry{
Editor: "unknown",
Os: "unknown",
}
}
ma := machineNames[entry.MachineNameId]
if ma == nil {
ma = &wakatime.MachineEntry{
Id: entry.MachineNameId,
Value: entry.MachineNameId,
}
}
return (&models.Heartbeat{
User: user,
UserID: user.ID,
Entity: entry.Entity,
Type: entry.Type,
Category: entry.Category,
Project: entry.Project,
Branch: entry.Branch,
Language: entry.Language,
IsWrite: entry.IsWrite,
Editor: ua.Editor,
OperatingSystem: ua.Os,
Machine: ma.Value,
2021-08-29 11:54:00 +03:00
UserAgent: ua.Value,
Time: entry.Time,
2021-02-06 02:31:30 +03:00
Origin: OriginWakatime,
OriginId: entry.Id,
}).Hashed()
}
func generateDays(from, to time.Time) []time.Time {
days := make([]time.Time, 0)
from = utils.StartOfDay(from)
to = utils.StartOfDay(to.AddDate(0, 0, 1))
for d := from; d.Before(to); d = d.AddDate(0, 0, 1) {
days = append(days, d)
}
return days
}