docs: instructions how to set up browser extension [skip ci]

chore: include more properties in status model for better compatibility

.github/workflows/ci.yml

@@ -75,33 +75,6 @@ jobs:
       with:
         sarif_file: mapi.sarif
 
-  build:
-    name: 'Build (Win, Linux, Mac)'
-
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.platform }}
-    env:
-      CGO_ENABLED: 0
-
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v3
-      with:
-        go-version: ^1.20
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
-    - name: Get dependencies
-      run: go get
-
-    - name: Build
-      run: go build -v .
-
   migration:
     name: Migration tests
     runs-on: ubuntu-latest

README.md

@@ -294,6 +294,22 @@ Preview:
 </details>
 <br>
 
+### Browser Plugin (Chrome & Firefox)
+The [browser-wakatime](https://github.com/wakatime/browser-wakatime) plugin enables you to track your web surfing in WakaTime (and Wakapi, of course). Visited websites will appear as "files" in the summary. Follow these instructions to get started:
+
+1. Install the browser extension from the official store ([Firefox](https://addons.mozilla.org/en-US/firefox/addon/wakatimes), [Chrome](https://chrome.google.com/webstore/detail/wakatime/jnbbnacmeggbgdjgaoojpmhdlkkpblgi?hl=de))
+2. Open the extension settings dialog
+3. Configure it like so (see screenshot below):
+    * API Key: Your personal API key (get it at [wakapi.dev](https://wakapi.dev))
+    * Logging Type: _Only the domain_
+    * API URL: `https://wakapi.dev/api/compat/wakatime/v1` (alternatively, replace _wakapi.dev_ with your self-hosted instance hostname)
+4. Save
+5. Start browsing!
+
+![](.github/assets/screenshot_browser_plugin.png)
+
+Note: the plugin will only sync heartbeats once in a while, so it might take some time for them to appear on Wakapi. To "force" it to sync, simply bring up the plugin main dialog.
+
 ## 📦 Data Export
 You can export your coding activity from Wakapi to CSV in the form of raw heartbeats. While there is no way to accomplish this directly through the web UI, we provide an easy-to-use Python [script](scripts/download_heartbeats.py) instead.
 

helpers/interval.go

@@ -0,0 +1,107 @@
+package helpers
+
+import (
+	"errors"
+	"github.com/muety/wakapi/models"
+	"github.com/muety/wakapi/utils"
+	"time"
+)
+
+func ParseInterval(interval string) (*models.IntervalKey, error) {
+	for _, i := range models.AllIntervals {
+		if i.HasAlias(interval) {
+			return i, nil
+		}
+	}
+	return nil, errors.New("not a valid interval")
+}
+
+func MustParseInterval(interval string) *models.IntervalKey {
+	key, _ := ParseInterval(interval)
+	return key
+}
+
+func MustResolveIntervalRawTZ(interval string, tz *time.Location) (from, to time.Time) {
+	_, from, to = ResolveIntervalRawTZ(interval, tz)
+	return from, to
+}
+
+func ResolveIntervalRawTZ(interval string, tz *time.Location) (err error, from, to time.Time) {
+	parsed, err := ParseInterval(interval)
+	if err != nil {
+		return err, time.Time{}, time.Time{}
+	}
+	return ResolveIntervalTZ(parsed, tz)
+}
+
+func ResolveIntervalTZ(interval *models.IntervalKey, tz *time.Location) (err error, from, to time.Time) {
+	now := time.Now().In(tz)
+	to = now
+
+	switch interval {
+	case models.IntervalToday:
+		from = utils.BeginOfToday(tz)
+	case models.IntervalYesterday:
+		from = utils.BeginOfToday(tz).Add(-24 * time.Hour)
+		to = utils.BeginOfToday(tz)
+	case models.IntervalPastDay:
+		from = now.Add(-24 * time.Hour)
+	case models.IntervalThisWeek:
+		from = utils.BeginOfThisWeek(tz)
+	case models.IntervalLastWeek:
+		from = utils.BeginOfThisWeek(tz).AddDate(0, 0, -7)
+		to = utils.BeginOfThisWeek(tz)
+	case models.IntervalThisMonth:
+		from = utils.BeginOfThisMonth(tz)
+	case models.IntervalLastMonth:
+		from = utils.BeginOfThisMonth(tz).AddDate(0, -1, 0)
+		to = utils.BeginOfThisMonth(tz)
+	case models.IntervalThisYear:
+		from = utils.BeginOfThisYear(tz)
+	case models.IntervalPast7Days:
+		from = now.AddDate(0, 0, -7)
+	case models.IntervalPast7DaysYesterday:
+		from = utils.BeginOfToday(tz).AddDate(0, 0, -1).AddDate(0, 0, -7)
+		to = utils.BeginOfToday(tz).AddDate(0, 0, -1)
+	case models.IntervalPast14Days:
+		from = now.AddDate(0, 0, -14)
+	case models.IntervalPast30Days:
+		from = now.AddDate(0, 0, -30)
+	case models.IntervalPast6Months:
+		from = now.AddDate(0, -6, 0)
+	case models.IntervalPast12Months:
+		from = now.AddDate(0, -12, 0)
+	case models.IntervalAny:
+		from = time.Time{}
+	default:
+		err = errors.New("invalid interval")
+	}
+
+	return err, from, to
+}
+
+// ResolveClosestRange returns the interval label (e.g. "last_7_days") of the maximum allowed range when having opted to share this many days or an error for days == 0.
+func ResolveMaximumRange(days int) (error, *models.IntervalKey) {
+	if days == 0 {
+		return errors.New("no matching interval"), nil
+	}
+	if days < 0 {
+		return nil, models.IntervalAny
+	}
+	if days < 7 {
+		return nil, models.IntervalPastDay
+	}
+	if days < 14 {
+		return nil, models.IntervalPast7Days
+	}
+	if days < 30 {
+		return nil, models.IntervalPast14Days
+	}
+	if days < 181 { // 3*31 + 2*30 + 1*28
+		return nil, models.IntervalPast30Days
+	}
+	if days < 365 { // 7*31 + 4*30 + 1*28
+		return nil, models.IntervalPast6Months
+	}
+	return nil, models.IntervalPast12Months
+}

helpers/interval_test.go

@@ -0,0 +1,27 @@
+package helpers
+
+import (
+	"github.com/muety/wakapi/models"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestResolveMaximumRange_Default(t *testing.T) {
+	for i := 1; i <= 366; i++ {
+		err1, maximumInterval := ResolveMaximumRange(i)
+		err2, from, to := ResolveIntervalTZ(maximumInterval, time.UTC)
+
+		assert.Nil(t, err1)
+		assert.Nil(t, err2)
+		assert.LessOrEqual(t, to.Sub(from), time.Duration(i*24)*time.Hour)
+	}
+}
+
+func TestResolveMaximumRange_EdgeCases(t *testing.T) {
+	err, _ := ResolveMaximumRange(0)
+	assert.NotNil(t, err)
+
+	_, maximumInterval := ResolveMaximumRange(-1)
+	assert.Equal(t, models.IntervalAny, maximumInterval)
+}

helpers/summary.go

@@ -3,7 +3,6 @@ package helpers
 import (
 	"errors"
 	"github.com/muety/wakapi/models"
-	"github.com/muety/wakapi/utils"
 	"net/http"
 	"time"
 )
@@ -82,69 +81,3 @@ func extractUser(r *http.Request) *models.User {
 	}
 	return nil
 }
-
-func ParseInterval(interval string) (*models.IntervalKey, error) {
-	for _, i := range models.AllIntervals {
-		if i.HasAlias(interval) {
-			return i, nil
-		}
-	}
-	return nil, errors.New("not a valid interval")
-}
-
-func MustResolveIntervalRawTZ(interval string, tz *time.Location) (from, to time.Time) {
-	_, from, to = ResolveIntervalRawTZ(interval, tz)
-	return from, to
-}
-
-func ResolveIntervalRawTZ(interval string, tz *time.Location) (err error, from, to time.Time) {
-	parsed, err := ParseInterval(interval)
-	if err != nil {
-		return err, time.Time{}, time.Time{}
-	}
-	return ResolveIntervalTZ(parsed, tz)
-}
-
-func ResolveIntervalTZ(interval *models.IntervalKey, tz *time.Location) (err error, from, to time.Time) {
-	now := time.Now().In(tz)
-	to = now
-
-	switch interval {
-	case models.IntervalToday:
-		from = utils.BeginOfToday(tz)
-	case models.IntervalYesterday:
-		from = utils.BeginOfToday(tz).Add(-24 * time.Hour)
-		to = utils.BeginOfToday(tz)
-	case models.IntervalThisWeek:
-		from = utils.BeginOfThisWeek(tz)
-	case models.IntervalLastWeek:
-		from = utils.BeginOfThisWeek(tz).AddDate(0, 0, -7)
-		to = utils.BeginOfThisWeek(tz)
-	case models.IntervalThisMonth:
-		from = utils.BeginOfThisMonth(tz)
-	case models.IntervalLastMonth:
-		from = utils.BeginOfThisMonth(tz).AddDate(0, -1, 0)
-		to = utils.BeginOfThisMonth(tz)
-	case models.IntervalThisYear:
-		from = utils.BeginOfThisYear(tz)
-	case models.IntervalPast7Days:
-		from = now.AddDate(0, 0, -7)
-	case models.IntervalPast7DaysYesterday:
-		from = utils.BeginOfToday(tz).AddDate(0, 0, -1).AddDate(0, 0, -7)
-		to = utils.BeginOfToday(tz).AddDate(0, 0, -1)
-	case models.IntervalPast14Days:
-		from = now.AddDate(0, 0, -14)
-	case models.IntervalPast30Days:
-		from = now.AddDate(0, 0, -30)
-	case models.IntervalPast6Months:
-		from = now.AddDate(0, -6, 0)
-	case models.IntervalPast12Months:
-		from = now.AddDate(0, -12, 0)
-	case models.IntervalAny:
-		from = time.Time{}
-	default:
-		err = errors.New("invalid interval")
-	}
-
-	return err, from, to
-}

migrations/20221028_fix_heartbeats_time_user_idx.go

@@ -1,12 +1,13 @@
 package migrations
 
 import (
+	"regexp"
+	"strings"
+
 	"github.com/emvi/logbuch"
 	"github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	"gorm.io/gorm"
-	"regexp"
-	"strings"
 )
 
 // due to an error in the model definition, idx_time_user used to only cover 'user_id', but not time column
@@ -37,7 +38,7 @@ func init() {
 				}
 
 				matches := regexp.MustCompile("(?i)\\((.+)\\)$").FindStringSubmatch(ddl)
-				if len(matches) > 0 && !strings.Contains(matches[0], "`user_id`") || !strings.Contains(matches[0], "`time`") {
+				if len(matches) > 0 && (!strings.Contains(matches[0], "`user_id`") || !strings.Contains(matches[0], "`time`")) {
 					drop = true
 				}
 			} else {

models/compat/wakatime/v1/data_dump.go

@@ -6,6 +6,10 @@ type DataDumpViewModel struct {
 	TotalPages int             `json:"total_pages"`
 }
 
+type DataDumpResultErrorModel struct {
+	Error string `json:"error"`
+}
+
 type DataDumpResultViewModel struct {
 	Data *DataDumpData `json:"data"`
 }

models/compat/wakatime/v1/stats.go

@@ -1,6 +1,7 @@
 package v1
 
 import (
+	"github.com/muety/wakapi/helpers"
 	"github.com/muety/wakapi/models"
 	"math"
 	"time"
@@ -18,9 +19,16 @@ type StatsData struct {
 	UserId                    string            `json:"user_id"`
 	Start                     time.Time         `json:"start"`
 	End                       time.Time         `json:"end"`
+	Status                    string            `json:"status"`
 	TotalSeconds              float64           `json:"total_seconds"`
 	DailyAverage              float64           `json:"daily_average"`
 	DaysIncludingHolidays     int               `json:"days_including_holidays"`
+	Range                     string            `json:"range"`
+	HumanReadableRange        string            `json:"human_readable_range"`
+	HumanReadableTotal        string            `json:"human_readable_total"`
+	HumanReadableDailyAverage string            `json:"human_readable_daily_average"`
+	IsCodingActivityVisible   bool              `json:"is_coding_activity_visible"`
+	IsOtherUsageVisible       bool              `json:"is_other_usage_visible"`
 	Editors                   []*SummariesEntry `json:"editors"`
 	Languages                 []*SummariesEntry `json:"languages"`
 	Machines                  []*SummariesEntry `json:"machines"`
@@ -38,11 +46,16 @@ func NewStatsFrom(summary *models.Summary, filters *models.Filters) *StatsViewMo
 		UserId:                summary.UserID,
 		Start:                 summary.FromTime.T(),
 		End:                   summary.ToTime.T(),
+		Status:                "ok",
 		TotalSeconds:          totalTime.Seconds(),
-		DailyAverage:          totalTime.Seconds() / float64(numDays),
 		DaysIncludingHolidays: numDays,
+		HumanReadableTotal:    helpers.FmtWakatimeDuration(totalTime),
 	}
 
+	if numDays > 0 {
+		data.DailyAverage = totalTime.Seconds() / float64(numDays)
+		data.HumanReadableDailyAverage = helpers.FmtWakatimeDuration(totalTime / time.Duration(numDays))
+	}
 	if math.IsInf(data.DailyAverage, 0) || math.IsNaN(data.DailyAverage) {
 		data.DailyAverage = 0
 	}

models/interval.go

@@ -1,27 +1,33 @@
 package models
 
+import (
+	"unicode"
+)
+
 // Support Wakapi and WakaTime range / interval identifiers
 // See https://wakatime.com/developers/#summaries
 var (
 	IntervalToday              = &IntervalKey{"today", "Today"}
 	IntervalYesterday          = &IntervalKey{"day", "yesterday", "Yesterday"}
+	IntervalPastDay            = &IntervalKey{"24_hours", "last_24_hours", "last_day", "Last 24 Hours"} // non-official one
 	IntervalThisWeek           = &IntervalKey{"week", "This Week"}
-	IntervalLastWeek           = &IntervalKey{"Last Week"}
+	IntervalLastWeek           = &IntervalKey{"last_week", "Last Week"}
 	IntervalThisMonth          = &IntervalKey{"month", "This Month"}
-	IntervalLastMonth          = &IntervalKey{"Last Month"}
-	IntervalThisYear           = &IntervalKey{"year"}
+	IntervalLastMonth          = &IntervalKey{"last_month", "Last Month"}
+	IntervalThisYear           = &IntervalKey{"year", "This Year"}
 	IntervalPast7Days          = &IntervalKey{"7_days", "last_7_days", "Last 7 Days"}
 	IntervalPast7DaysYesterday = &IntervalKey{"Last 7 Days from Yesterday"}
-	IntervalPast14Days         = &IntervalKey{"Last 14 Days"}
+	IntervalPast14Days         = &IntervalKey{"14_days", "last_14_days", "Last 14 Days"}
 	IntervalPast30Days         = &IntervalKey{"30_days", "last_30_days", "Last 30 Days"}
-	IntervalPast6Months        = &IntervalKey{"6_months", "last_6_months"}
-	IntervalPast12Months       = &IntervalKey{"12_months", "last_12_months", "last_year"}
-	IntervalAny                = &IntervalKey{"any", "all_time"}
+	IntervalPast6Months        = &IntervalKey{"6_months", "last_6_months", "Last 6 Months"}
+	IntervalPast12Months       = &IntervalKey{"12_months", "last_12_months", "last_year", "Last 12 Months"}
+	IntervalAny                = &IntervalKey{"any", "all_time", "All Time"}
 )
 
 var AllIntervals = []*IntervalKey{
 	IntervalToday,
 	IntervalYesterday,
+	IntervalPastDay,
 	IntervalThisWeek,
 	IntervalLastWeek,
 	IntervalThisMonth,
@@ -46,3 +52,12 @@ func (k *IntervalKey) HasAlias(s string) bool {
 	}
 	return false
 }
+
+func (k *IntervalKey) GetHumanReadable() string {
+	for _, s := range *k {
+		if unicode.IsUpper(rune(s[0])) {
+			return s
+		}
+	}
+	return ""
+}

models/user.go

@@ -163,6 +163,10 @@ func (u *User) MinDataAge() time.Time {
 	return time.Now().AddDate(0, -retentionMonths, 0)
 }
 
+func (u *User) AnyDataShared() bool {
+	return u.ShareDataMaxDays != 0 && (u.ShareEditors || u.ShareLanguages || u.ShareProjects || u.ShareOSs || u.ShareMachines || u.ShareLabels)
+}
+
 func (c *CredentialsReset) IsValid() bool {
 	return ValidatePassword(c.PasswordNew) &&
 		c.PasswordNew == c.PasswordRepeat

routes/compat/wakatime/v1/stats.go

@@ -76,8 +76,14 @@ func (h *StatsHandler) Get(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// if no range was requested, get the maximum allowed range given the users max shared days, otherwise default to past 7 days (which will fail in the next step, because user didn't allow any sharing)
+	// this "floors" the user's maximum shared date to the supported range buckets (e.g. if user opted to share 12 days, we'll still fallback to "last_7_days") for consistency with wakatime
 	if rangeParam == "" {
-		rangeParam = (*models.IntervalPast7Days)[0]
+		if _, userRange := helpers.ResolveMaximumRange(requestedUser.ShareDataMaxDays); userRange != nil {
+			rangeParam = (*userRange)[1]
+		} else {
+			rangeParam = (*models.IntervalPast7Days)[1]
+		}
 	}
 
 	err, rangeFrom, rangeTo := helpers.ResolveIntervalRawTZ(rangeParam, requestedUser.TZ())
@@ -103,6 +109,10 @@ func (h *StatsHandler) Get(w http.ResponseWriter, r *http.Request) {
 	}
 
 	stats := v1.NewStatsFrom(summary, &models.Filters{})
+	stats.Data.Range = rangeParam
+	stats.Data.HumanReadableRange = helpers.MustParseInterval(rangeParam).GetHumanReadable()
+	stats.Data.IsCodingActivityVisible = requestedUser.ShareDataMaxDays != 0
+	stats.Data.IsOtherUsageVisible = requestedUser.AnyDataShared()
 
 	// post filter stats according to user's given sharing permissions
 	if !requestedUser.ShareEditors {

services/imports/wakatime_dump.go

@@ -6,15 +6,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/duke-git/lancet/v2/slice"
+	"github.com/muety/wakapi/utils"
+	"net/http"
+	"time"
+
 	"github.com/emvi/logbuch"
 	"github.com/muety/artifex/v2"
 	"github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
-	"github.com/muety/wakapi/utils"
-	"net/http"
-	"time"
 )
 
 // data example: https://github.com/muety/wakapi/issues/323#issuecomment-1627467052
@@ -40,20 +40,28 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 	url := config.WakatimeApiUrl + config.WakatimeApiDataDumpUrl // this importer only works with wakatime currently, so no point in using user's custom wakatime api url
 	req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(`{ "type": "heartbeats", "email_when_finished": false }`)))
 	res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
-	if err != nil {
+
+	if err != nil && res.StatusCode == http.StatusBadRequest {
+		var datadumpError wakatime.DataDumpResultErrorModel
+		if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil {
+			return nil, err
+		}
+		// in case of this error message, a dump had already been requested before and can simply be downloaded now
+		// -> just keep going as usual (kick off poll loop), otherwise yield error
+		if datadumpError.Error == "Wait for your current export to expire before creating another." {
+			logbuch.Info("failed to request new dump, because other non-expired dump already existing, using that one")
+		} else {
+			return nil, err
+		}
+	} else if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
 
-	var datadumpData wakatime.DataDumpResultViewModel
-	if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil {
-		return nil, err
-	}
-
 	var readyPollTimer *artifex.DispatchTicker
 
 	// callbacks
-	checkDumpReady := func(dumpId string, user *models.User) (bool, *wakatime.DataDumpData, error) {
+	checkDumpAvailable := func(user *models.User) (bool, *wakatime.DataDumpData, error) {
 		req, _ := http.NewRequest(http.MethodGet, url, nil)
 		res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
 		if err != nil {
@@ -65,14 +73,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 			return false, nil, err
 		}
 
-		dump, ok := slice.FindBy[*wakatime.DataDumpData](datadumpData.Data, func(i int, item *wakatime.DataDumpData) bool {
-			return item.Id == dumpId
-		})
-		if !ok {
-			return false, nil, errors.New(fmt.Sprintf("data dump with id '%s' for user '%s' not found", dumpId, user.ID))
+		if len(datadumpData.Data) < 1 {
+			return false, nil, errors.New("no dumps available")
 		}
 
-		return dump.Status == "Completed", dump, nil
+		return datadumpData.Data[0].Status == "Completed", datadumpData.Data[0], nil
 	}
 
 	onDumpFailed := func(err error, user *models.User) {
@@ -131,11 +136,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 	// start polling for dump to be ready
 	readyPollTimer, err = w.queue.DispatchEvery(func() {
 		u := *user
-		ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u)
-		logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete)
+		ok, dump, err := checkDumpAvailable(&u)
 		if err != nil {
 			onDumpFailed(err, &u)
 		} else if ok {
+			logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete)
 			onDumpReady(dump, &u, out)
 		}
 	}, 10*time.Second)

utils/http.go

@@ -1,8 +1,10 @@
 package utils
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -92,7 +94,15 @@ func RaiseForStatus(res *http.Response, err error) (*http.Response, error) {
 		return res, err
 	}
 	if res.StatusCode >= 400 {
-		return res, fmt.Errorf("got response status %d for '%s %s'", res.StatusCode, res.Request.Method, res.Request.URL.String())
+		message := "<body omitted or empty>"
+		contentType := res.Header.Get("content-type")
+		if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "application/json") {
+			body, _ := io.ReadAll(res.Body)
+			res.Body.Close()
+			res.Body = io.NopCloser(bytes.NewBuffer(body))
+			message = string(body)
+		}
+		return res, fmt.Errorf("got response status %d for '%s %s' - %s", res.StatusCode, res.Request.Method, res.Request.URL.String(), message)
 	}
 	return res, nil
 }