Merge remote-tracking branch 'origin/master'

Simplify import checks

.github/workflows/ci.yml

@@ -75,33 +75,6 @@ jobs:
       with:
         sarif_file: mapi.sarif
 
-  build:
-    name: 'Build (Win, Linux, Mac)'
-
-    strategy:
-      matrix:
-        platform: [ubuntu-latest, macos-latest, windows-latest]
-
-    runs-on: ${{ matrix.platform }}
-    env:
-      CGO_ENABLED: 0
-
-    steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v3
-      with:
-        go-version: ^1.20
-      id: go
-
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v3
-
-    - name: Get dependencies
-      run: go get
-
-    - name: Build
-      run: go build -v .
-
   migration:
     name: Migration tests
     runs-on: ubuntu-latest

models/compat/wakatime/v1/data_dump.go

@@ -6,6 +6,10 @@ type DataDumpViewModel struct {
 	TotalPages int             `json:"total_pages"`
 }
 
+type DataDumpResultErrorModel struct {
+	Error string `json:"error"`
+}
+
 type DataDumpResultViewModel struct {
 	Data *DataDumpData `json:"data"`
 }

services/imports/wakatime_dump.go

@@ -6,15 +6,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"github.com/duke-git/lancet/v2/slice"
+	"github.com/muety/wakapi/utils"
+	"net/http"
+	"time"
+
 	"github.com/emvi/logbuch"
 	"github.com/muety/artifex/v2"
 	"github.com/muety/wakapi/config"
 	"github.com/muety/wakapi/models"
 	wakatime "github.com/muety/wakapi/models/compat/wakatime/v1"
-	"github.com/muety/wakapi/utils"
-	"net/http"
-	"time"
 )
 
 // data example: https://github.com/muety/wakapi/issues/323#issuecomment-1627467052
@@ -40,20 +40,28 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 	url := config.WakatimeApiUrl + config.WakatimeApiDataDumpUrl // this importer only works with wakatime currently, so no point in using user's custom wakatime api url
 	req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(`{ "type": "heartbeats", "email_when_finished": false }`)))
 	res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
-	if err != nil {
+
+	if err != nil && res.StatusCode == http.StatusBadRequest {
+		var datadumpError wakatime.DataDumpResultErrorModel
+		if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil {
+			return nil, err
+		}
+		// in case of this error message, a dump had already been requested before and can simply be downloaded now
+		// -> just keep going as usual (kick off poll loop), otherwise yield error
+		if datadumpError.Error == "Wait for your current export to expire before creating another." {
+			logbuch.Info("failed to request new dump, because other non-expired dump already existing, using that one")
+		} else {
+			return nil, err
+		}
+	} else if err != nil {
 		return nil, err
 	}
 	defer res.Body.Close()
 
-	var datadumpData wakatime.DataDumpResultViewModel
-	if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil {
-		return nil, err
-	}
-
 	var readyPollTimer *artifex.DispatchTicker
 
 	// callbacks
-	checkDumpReady := func(dumpId string, user *models.User) (bool, *wakatime.DataDumpData, error) {
+	checkDumpAvailable := func(user *models.User) (bool, *wakatime.DataDumpData, error) {
 		req, _ := http.NewRequest(http.MethodGet, url, nil)
 		res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req)))
 		if err != nil {
@@ -65,14 +73,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 			return false, nil, err
 		}
 
-		dump, ok := slice.FindBy[*wakatime.DataDumpData](datadumpData.Data, func(i int, item *wakatime.DataDumpData) bool {
+		if len(datadumpData.Data) < 1 {
-			return item.Id == dumpId
+			return false, nil, errors.New("no dumps available")
-		})
-		if !ok {
-			return false, nil, errors.New(fmt.Sprintf("data dump with id '%s' for user '%s' not found", dumpId, user.ID))
 		}
 
-		return dump.Status == "Completed", dump, nil
+		return datadumpData.Data[0].Status == "Completed", datadumpData.Data[0], nil
 	}
 
 	onDumpFailed := func(err error, user *models.User) {
@@ -131,11 +136,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT
 	// start polling for dump to be ready
 	readyPollTimer, err = w.queue.DispatchEvery(func() {
 		u := *user
-		ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u)
+		ok, dump, err := checkDumpAvailable(&u)
-		logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete)
 		if err != nil {
 			onDumpFailed(err, &u)
 		} else if ok {
+			logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete)
 			onDumpReady(dump, &u, out)
 		}
 	}, 10*time.Second)

utils/http.go

@@ -1,8 +1,10 @@
 package utils
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -92,7 +94,15 @@ func RaiseForStatus(res *http.Response, err error) (*http.Response, error) {
 		return res, err
 	}
 	if res.StatusCode >= 400 {
-		return res, fmt.Errorf("got response status %d for '%s %s'", res.StatusCode, res.Request.Method, res.Request.URL.String())
+		message := "<body omitted or empty>"
+		contentType := res.Header.Get("content-type")
+		if strings.HasPrefix(contentType, "text/") || strings.HasPrefix(contentType, "application/json") {
+			body, _ := io.ReadAll(res.Body)
+			res.Body.Close()
+			res.Body = io.NopCloser(bytes.NewBuffer(body))
+			message = string(body)
+		}
+		return res, fmt.Errorf("got response status %d for '%s %s' - %s", res.StatusCode, res.Request.Method, res.Request.URL.String(), message)
 	}
 	return res, nil
 }