1
0
mirror of https://github.com/muety/wakapi.git synced 2023-08-10 21:12:56 +03:00

Compare commits

...

12 Commits

Author SHA1 Message Date
05bc55a488 docs: instructions how to set up browser extension [skip ci] 2023-07-30 08:17:14 +02:00
a9364e3d9e Merge remote-tracking branch 'origin/master' 2023-07-28 12:08:54 +02:00
ec65847d0c fix: make stats endpoint default to user-chosen time range (resolve #508)
chore: include more properties in status model for better compatibility
2023-07-28 12:08:47 +02:00
eca443be35 Merge pull request #507 from cbrand/master
fix failing migration which prohibits startup
2023-07-21 08:50:42 +02:00
04ec44dcef fix: failing migration
Fix an issue in migration which results in the following error message
due to wrong and or precedence configuration:
```
panic: runtime error: index out of range [0] with length 0
```
2023-07-20 18:44:26 +02:00
938290b2da Merge remote-tracking branch 'origin/master' 2023-07-19 18:36:35 +02:00
c8b88ccef5 chore: log response body of failed http requests 2023-07-19 18:36:27 +02:00
bc2d05bd85 ci: skip multi-platform build step on pushes and prs [skip ci] 2023-07-14 08:50:23 +02:00
3785867c3a Merge pull request #504 from muety/502-imports
Simplify import checks
2023-07-14 08:47:54 +02:00
56de275781 chore: simplify import checks
fix: minor fixes
2023-07-13 20:48:56 +02:00
583ddcab7a refactor: remove repeated code in readyPollTimer 2023-07-14 00:33:55 +08:00
7b0bbcefe6 fix(import): data dump already exists
handle the import when there is already an active data dump exists.

Resolves #502
2023-07-13 23:54:48 +08:00
15 changed files with 953 additions and 738 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

View File

@ -75,33 +75,6 @@ jobs:
with:
sarif_file: mapi.sarif
build:
name: 'Build (Win, Linux, Mac)'
strategy:
matrix:
platform: [ubuntu-latest, macos-latest, windows-latest]
runs-on: ${{ matrix.platform }}
env:
CGO_ENABLED: 0
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v3
with:
go-version: ^1.20
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v3
- name: Get dependencies
run: go get
- name: Build
run: go build -v .
migration:
name: Migration tests
runs-on: ubuntu-latest

View File

@ -294,6 +294,22 @@ Preview:
</details>
<br>
### Browser Plugin (Chrome & Firefox)
The [browser-wakatime](https://github.com/wakatime/browser-wakatime) plugin enables you to track your web surfing in WakaTime (and Wakapi, of course). Visited websites will appear as "files" in the summary. Follow these instructions to get started:
1. Install the browser extension from the official store ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/wakatimes), [Chrome](https://chrome.google.com/webstore/detail/wakatime/jnbbnacmeggbgdjgaoojpmhdlkkpblgi?hl=de))
2. Open the extension settings dialog
3. Configure it like so (see screenshot below):
* API Key: Your personal API key (get it at [wakapi.dev](https://wakapi.dev))
* Logging Type: _Only the domain_
* API URL: `https://wakapi.dev/api/compat/wakatime/v1` (alternatively, replace _wakapi.dev_ with your self-hosted instance hostname)
4. Save
5. Start browsing!
![](.github/assets/screenshot_browser_plugin.png)
Note: the plugin will only sync heartbeats once in a while, so it might take some time for them to appear on Wakapi. To "force" it to sync, simply bring up the plugin main dialog.
## 📦 Data Export
You can export your coding activity from Wakapi to CSV in the form of raw heartbeats. While there is no way to accomplish this directly through the web UI, we provide an easy-to-use Python [script](scripts/download_heartbeats.py) instead.

File diff suppressed because it is too large Load Diff

107
helpers/interval.go Normal file
View File

@ -0,0 +1,107 @@
package helpers
import (
"errors"
"github.com/muety/wakapi/models"
"github.com/muety/wakapi/utils"
"time"
)
func ParseInterval(interval string) (*models.IntervalKey, error) {
for _, i := range models.AllIntervals {
if i.HasAlias(interval) {
return i, nil
}
}
return nil, errors.New("not a valid interval")
}
func MustParseInterval(interval string) *models.IntervalKey {
key, _ := ParseInterval(interval)
return key
}
func MustResolveIntervalRawTZ(interval string, tz *time.Location) (from, to time.Time) {
_, from, to = ResolveIntervalRawTZ(interval, tz)
return from, to
}
func ResolveIntervalRawTZ(interval string, tz *time.Location) (err error, from, to time.Time) {
parsed, err := ParseInterval(interval)
if err != nil {
return err, time.Time{}, time.Time{}
}
return ResolveIntervalTZ(parsed, tz)
}
func ResolveIntervalTZ(interval *models.IntervalKey, tz *time.Location) (err error, from, to time.Time) {
now := time.Now().In(tz)
to = now
switch interval {
case models.IntervalToday:
from = utils.BeginOfToday(tz)
case models.IntervalYesterday:
from = utils.BeginOfToday(tz).Add(-24 * time.Hour)
to = utils.BeginOfToday(tz)
case models.IntervalPastDay:
from = now.Add(-24 * time.Hour)
case models.IntervalThisWeek:
from = utils.BeginOfThisWeek(tz)
case models.IntervalLastWeek:
from = utils.BeginOfThisWeek(tz).AddDate(0, 0, -7)
to = utils.BeginOfThisWeek(tz)
case models.IntervalThisMonth:
from = utils.BeginOfThisMonth(tz)
case models.IntervalLastMonth:
from = utils.BeginOfThisMonth(tz).AddDate(0, -1, 0)
to = utils.BeginOfThisMonth(tz)
case models.IntervalThisYear:
from = utils.BeginOfThisYear(tz)
case models.IntervalPast7Days:
from = now.AddDate(0, 0, -7)
case models.IntervalPast7DaysYesterday:
from = utils.BeginOfToday(tz).AddDate(0, 0, -1).AddDate(0, 0, -7)
to = utils.BeginOfToday(tz).AddDate(0, 0, -1)
case models.IntervalPast14Days:
from = now.AddDate(0, 0, -14)
case models.IntervalPast30Days:
from = now.AddDate(0, 0, -30)
case models.IntervalPast6Months:
from = now.AddDate(0, -6, 0)
case models.IntervalPast12Months:
from = now.AddDate(0, -12, 0)
case models.IntervalAny:
from = time.Time{}
default:
err = errors.New("invalid interval")
}
return err, from, to
}
// ResolveClosestRange returns the interval label (e.g. "last_7_days") of the maximum allowed range when having opted to share this many days or an error for days == 0.
func ResolveMaximumRange(days int) (error, *models.IntervalKey) {
if days == 0 {
return errors.New("no matching interval"), nil
}
if days < 0 {
return nil, models.IntervalAny
}
if days < 7 {
return nil, models.IntervalPastDay
}
if days < 14 {
return nil, models.IntervalPast7Days
}
if days < 30 {
return nil, models.IntervalPast14Days
}
if days < 181 { // 3*31 + 2*30 + 1*28
return nil, models.IntervalPast30Days
}
if days < 365 { // 7*31 + 4*30 + 1*28
return nil, models.IntervalPast6Months
}
return nil, models.IntervalPast12Months
}

27
helpers/interval_test.go Normal file
View File

@ -0,0 +1,27 @@
package helpers
import (
"github.com/muety/wakapi/models"
"github.com/stretchr/testify/assert"
"testing"
"time"
)
func TestResolveMaximumRange_Default(t *testing.T) {
for i := 1; i <= 366; i++ {
err1, maximumInterval := ResolveMaximumRange(i)
err2, from, to := ResolveIntervalTZ(maximumInterval, time.UTC)
assert.Nil(t, err1)
assert.Nil(t, err2)
assert.LessOrEqual(t, to.Sub(from), time.Duration(i*24)*time.Hour)
}
}
func TestResolveMaximumRange_EdgeCases(t *testing.T) {
err, _ := ResolveMaximumRange(0)
assert.NotNil(t, err)
_, maximumInterval := ResolveMaximumRange(-1)
assert.Equal(t, models.IntervalAny, maximumInterval)
}

View File

@ -3,7 +3,6 @@ package helpers
import (
"errors"
"github.com/muety/wakapi/models"
"github.com/muety/wakapi/utils"
"net/http"
"time"
)
@ -82,69 +81,3 @@ func extractUser(r *http.Request) *models.User {
}
return nil
}
func ParseInterval(interval string) (*models.IntervalKey, error) {
for _, i := range models.AllIntervals {
if i.HasAlias(interval) {
return i, nil
}
}
return nil, errors.New("not a valid interval")
}
func MustResolveIntervalRawTZ(interval string, tz *time.Location) (from, to time.Time) {
_, from, to = ResolveIntervalRawTZ(interval, tz)
return from, to
}
func ResolveIntervalRawTZ(interval string, tz *time.Location) (err error, from, to time.Time) {
parsed, err := ParseInterval(interval)
if err != nil {
return err, time.Time{}, time.Time{}
}
return ResolveIntervalTZ(parsed, tz)
}
func ResolveIntervalTZ(interval *models.IntervalKey, tz *time.Location) (err error, from, to time.Time) {
now := time.Now().In(tz)
to = now
switch interval {
case models.IntervalToday:
from = utils.BeginOfToday(tz)
case models.IntervalYesterday:
from = utils.BeginOfToday(tz).Add(-24 * time.Hour)
to = utils.BeginOfToday(tz)
case models.IntervalThisWeek:
from = utils.BeginOfThisWeek(tz)
case models.IntervalLastWeek:
from = utils.BeginOfThisWeek(tz).AddDate(0, 0, -7)
to = utils.BeginOfThisWeek(tz)
case models.IntervalThisMonth:
from = utils.BeginOfThisMonth(tz)
case models.IntervalLastMonth:
from = utils.BeginOfThisMonth(tz).AddDate(0, -1, 0)
to = utils.BeginOfThisMonth(tz)
case models.IntervalThisYear:
from = utils.BeginOfThisYear(tz)
case models.IntervalPast7Days:
from = now.AddDate(0, 0, -7)
case models.IntervalPast7DaysYesterday:
from = utils.BeginOfToday(tz).AddDate(0, 0, -1).AddDate(0, 0, -7)
to = utils.BeginOfToday(tz).AddDate(0, 0, -1)
case models.IntervalPast14Days:
from = now.AddDate(0, 0, -14)
case models.IntervalPast30Days:
from = now.AddDate(0, 0, -30)
case models.IntervalPast6Months:
from = now.AddDate(0, -6, 0)
case models.IntervalPast12Months:
from = now.AddDate(0, -12, 0)
case models.IntervalAny:
from = time.Time{}
default:
err = errors.New("invalid interval")
}
return err, from, to
}

View File

@ -1,12 +1,13 @@
package migrations
import (
"regexp"
"strings"
"github.com/emvi/logbuch"
"github.com/muety/wakapi/config"
"github.com/muety/wakapi/models"
"gorm.io/gorm"
"regexp"
"strings"
)
// due to an error in the model definition, idx_time_user used to only cover 'user_id', but not time column
@ -37,7 +38,7 @@ func init() {
}
matches := regexp.MustCompile("(?i)\\((.+)\\)$").FindStringSubmatch(ddl)
if len(matches) > 0 && !strings.Contains(matches[0], "`user_id`") || !strings.Contains(matches[0], "`time`") {
if len(matches) > 0 && (!strings.Contains(matches[0], "`user_id`") || !strings.Contains(matches[0], "`time`")) {
drop = true
}
} else {

View File

@ -6,6 +6,10 @@ type DataDumpViewModel struct {
TotalPages int `json:"total_pages"`
}
type DataDumpResultErrorModel struct {
Error string `json:"error"`
}
type DataDumpResultViewModel struct {
Data *DataDumpData `json:"data"`
}

View File

@ -1,6 +1,7 @@
package v1
import (
"github.com/muety/wakapi/helpers"
"github.com/muety/wakapi/models"
"math"
"time"
@ -14,19 +15,26 @@ type StatsViewModel struct {
}
type StatsData struct {
Username string `json:"username"`
UserId string `json:"user_id"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
TotalSeconds float64 `json:"total_seconds"`
DailyAverage float64 `json:"daily_average"`
DaysIncludingHolidays int `json:"days_including_holidays"`
Editors []*SummariesEntry `json:"editors"`
Languages []*SummariesEntry `json:"languages"`
Machines []*SummariesEntry `json:"machines"`
Projects []*SummariesEntry `json:"projects"`
OperatingSystems []*SummariesEntry `json:"operating_systems"`
Branches []*SummariesEntry `json:"branches,omitempty"`
Username string `json:"username"`
UserId string `json:"user_id"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
Status string `json:"status"`
TotalSeconds float64 `json:"total_seconds"`
DailyAverage float64 `json:"daily_average"`
DaysIncludingHolidays int `json:"days_including_holidays"`
Range string `json:"range"`
HumanReadableRange string `json:"human_readable_range"`
HumanReadableTotal string `json:"human_readable_total"`
HumanReadableDailyAverage string `json:"human_readable_daily_average"`
IsCodingActivityVisible bool `json:"is_coding_activity_visible"`
IsOtherUsageVisible bool `json:"is_other_usage_visible"`
Editors []*SummariesEntry `json:"editors"`
Languages []*SummariesEntry `json:"languages"`
Machines []*SummariesEntry `json:"machines"`
Projects []*SummariesEntry `json:"projects"`
OperatingSystems []*SummariesEntry `json:"operating_systems"`
Branches []*SummariesEntry `json:"branches,omitempty"`
}
func NewStatsFrom(summary *models.Summary, filters *models.Filters) *StatsViewModel {
@ -38,11 +46,16 @@ func NewStatsFrom(summary *models.Summary, filters *models.Filters) *StatsViewMo
UserId: summary.UserID,
Start: summary.FromTime.T(),
End: summary.ToTime.T(),
Status: "ok",
TotalSeconds: totalTime.Seconds(),
DailyAverage: totalTime.Seconds() / float64(numDays),
DaysIncludingHolidays: numDays,
HumanReadableTotal: helpers.FmtWakatimeDuration(totalTime),
}
if numDays > 0 {
data.DailyAverage = totalTime.Seconds() / float64(numDays)
data.HumanReadableDailyAverage = helpers.FmtWakatimeDuration(totalTime / time.Duration(numDays))
}
if math.IsInf(data.DailyAverage, 0) || math.IsNaN(data.DailyAverage) {
data.DailyAverage = 0
}

View File

@ -1,27 +1,33 @@
package models
import (
"unicode"
)
// Support Wakapi and WakaTime range / interval identifiers
// See https://wakatime.com/developers/#summaries
var (
IntervalToday = &IntervalKey{"today", "Today"}
IntervalYesterday = &IntervalKey{"day", "yesterday", "Yesterday"}
IntervalPastDay = &IntervalKey{"24_hours", "last_24_hours", "last_day", "Last 24 Hours"} // non-official one
IntervalThisWeek = &IntervalKey{"week", "This Week"}
IntervalLastWeek = &IntervalKey{"Last Week"}
IntervalLastWeek = &IntervalKey{"last_week", "Last Week"}
IntervalThisMonth = &IntervalKey{"month", "This Month"}
IntervalLastMonth = &IntervalKey{"Last Month"}
IntervalThisYear = &IntervalKey{"year"}
IntervalLastMonth = &IntervalKey{"last_month", "Last Month"}
IntervalThisYear = &IntervalKey{"year", "This Year"}
IntervalPast7Days = &IntervalKey{"7_days", "last_7_days", "Last 7 Days"}
IntervalPast7DaysYesterday = &IntervalKey{"Last 7 Days from Yesterday"}
IntervalPast14Days = &IntervalKey{"Last 14 Days"}
IntervalPast14Days = &IntervalKey{"14_days", "last_14_days", "Last 14 Days"}
IntervalPast30Days = &IntervalKey{"30_days", "last_30_days", "Last 30 Days"}
IntervalPast6Months = &IntervalKey{"6_months", "last_6_months"}
IntervalPast12Months = &IntervalKey{"12_months", "last_12_months", "last_year"}
IntervalAny = &IntervalKey{"any", "all_time"}
IntervalPast6Months = &IntervalKey{"6_months", "last_6_months", "Last 6 Months"}
IntervalPast12Months = &IntervalKey{"12_months", "last_12_months", "last_year", "Last 12 Months"}
IntervalAny = &IntervalKey{"any", "all_time", "All Time"}
)
var AllIntervals = []*IntervalKey{
IntervalToday,
IntervalYesterday,
IntervalPastDay,
IntervalThisWeek,
IntervalLastWeek,
IntervalThisMonth,
@ -46,3 +52,12 @@ func (k *IntervalKey) HasAlias(s string) bool {
}
return false
}
func (k *IntervalKey) GetHumanReadable() string {
for _, s := range *k {
if unicode.IsUpper(rune(s[0])) {
return s
}
}
return ""
}

View File

@ -163,6 +163,10 @@ func (u *User) MinDataAge() time.Time {
return time.Now().AddDate(0, -retentionMonths, 0)
}
func (u *User) AnyDataShared() bool {
return u.ShareDataMaxDays != 0 && (u.ShareEditors || u.ShareLanguages || u.ShareProjects || u.ShareOSs || u.ShareMachines || u.ShareLabels)
}
func (c *CredentialsReset) IsValid() bool {
return ValidatePassword(c.PasswordNew) &&
c.PasswordNew == c.PasswordRepeat

View File

@ -76,8 +76,14 @@ func (h *StatsHandler) Get(w http.ResponseWriter, r *http.Request) {
return
}
// if no range was requested, get the maximum allowed range given the users max shared days, otherwise default to past 7 days (which will fail in the next step, because user didn't allow any sharing)
// this "floors" the user's maximum shared date to the supported range buckets (e.g. if user opted to share 12 days, we'll still fallback to "last_7_days") for consistency with wakatime
if rangeParam == "" {
rangeParam = (*models.IntervalPast7Days)[0]
if _, userRange := helpers.ResolveMaximumRange(requestedUser.ShareDataMaxDays); userRange != nil {
rangeParam = (*userRange)[1]
} else {
rangeParam = (*models.IntervalPast7Days)[1]
}
}
err, rangeFrom, rangeTo := helpers.ResolveIntervalRawTZ(rangeParam, requestedUser.TZ())
@ -103,6 +109,10 @@ func (h *StatsHandler) Get(w http.ResponseWriter, r *http.Request) {
}
stats := v1.NewStatsFrom(summary, &models.Filters{})
stats.Data.Range = rangeParam
stats.Data.HumanReadableRange = helpers.MustParseInterval(rangeParam).GetHumanReadable()
stats.Data.IsCodingActivityVisible = requestedUser.ShareDataMaxDays != 0
stats.Data.IsOtherUsageVisible = requestedUser.AnyDataShared()
// post filter stats according to user's given sharing permissions
if !requestedUser.ShareEditors {

View File

@ -6,15 +6,15 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/duke-git/lancet/v2/slice"
"github.com/muety/wakapi/utils"
"net/http"
"time"
"github.com/emvi/logbuch"
"github.com/muety/artifex/v2"
"github.com/muety/wakapi/config"
"github.com/muety/wakapi/models"
wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
"github.com/muety/wakapi/utils"
"net/http"
"time"
)
// data example: https://github.com/muety/wakapi/issues/323#issuecomment-1627467052
@ -40,20 +40,28 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
url := config.WakatimeApiUrl + config.WakatimeApiDataDumpUrl // this importer only works with wakatime currently, so no point in using user's custom wakatime api url
req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(`{ "type": "heartbeats", "email_when_finished": false }`)))
res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
if err != nil {
if err != nil && res.StatusCode == http.StatusBadRequest {
var datadumpError wakatime.DataDumpResultErrorModel
if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil {
return nil, err
}
// in case of this error message, a dump had already been requested before and can simply be downloaded now
// -> just keep going as usual (kick off poll loop), otherwise yield error
if datadumpError.Error == "Wait for your current export to expire before creating another." {
logbuch.Info("failed to request new dump, because other non-expired dump already existing, using that one")
} else {
return nil, err
}
} else if err != nil {
return nil, err
}
defer res.Body.Close()
var datadumpData wakatime.DataDumpResultViewModel
if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil {
return nil, err
}
var readyPollTimer *artifex.DispatchTicker
// callbacks
checkDumpReady := func(dumpId string, user *models.User) (bool, *wakatime.DataDumpData, error) {
checkDumpAvailable := func(user *models.User) (bool, *wakatime.DataDumpData, error) {
req, _ := http.NewRequest(http.MethodGet, url, nil)
res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
if err != nil {
@ -65,14 +73,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
return false, nil, err
}
dump, ok := slice.FindBy[*wakatime.DataDumpData](datadumpData.Data, func(i int, item *wakatime.DataDumpData) bool {
return item.Id == dumpId
})
if !ok {
return false, nil, errors.New(fmt.Sprintf("data dump with id '%s' for user '%s' not found", dumpId, user.ID))
if len(datadumpData.Data) < 1 {
return false, nil, errors.New("no dumps available")
}
return dump.Status == "Completed", dump, nil
return datadumpData.Data[0].Status == "Completed", datadumpData.Data[0], nil
}
onDumpFailed := func(err error, user *models.User) {
@ -131,11 +136,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
// start polling for dump to be ready
readyPollTimer, err = w.queue.DispatchEvery(func() {
u := *user
ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u)
logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete)
ok, dump, err := checkDumpAvailable(&u)
if err != nil {
onDumpFailed(err, &u)
} else if ok {
logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete)
onDumpReady(dump, &u, out)
}
}, 10*time.Second)

View File

@ -1,8 +1,10 @@
package utils
import (
"bytes"
"errors"
"fmt"
"io"
"net/http"
"regexp"
"strconv"
@ -92,7 +94,15 @@ func RaiseForStatus(res *http.Response, err error) (*http.Response, error) {
return res, err
}
if res.StatusCode >= 400 {
return res, fmt.Errorf("got response status %d for '%s %s'", res.StatusCode, res.Request.Method, res.Request.URL.String())
message := "<body omitted or empty>"
contentType := res.Header.Get("content-type")
if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "application/json") {
body, _ := io.ReadAll(res.Body)
res.Body.Close()
res.Body = io.NopCloser(bytes.NewBuffer(body))
message = string(body)
}
return res, fmt.Errorf("got response status %d for '%s %s' - %s", res.StatusCode, res.Request.Method, res.Request.URL.String(), message)
}
return res, nil
}