From 7b0bbcefe628fb404aedc5db41e2a7d1a7ea5c11 Mon Sep 17 00:00:00 2001 From: Edward <73746306+WangEdward@users.noreply.github.com> Date: Thu, 13 Jul 2023 23:54:48 +0800 Subject: [PATCH 1/4] fix(import): data dump already exists handle the import when there is already an active data dump exists. Resolves #502 --- models/compat/wakatime/v1/data_dump.go | 4 ++ services/imports/wakatime_dump.go | 62 +++++++++++++++++++++----- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/models/compat/wakatime/v1/data_dump.go b/models/compat/wakatime/v1/data_dump.go index d788df0..07c95ba 100644 --- a/models/compat/wakatime/v1/data_dump.go +++ b/models/compat/wakatime/v1/data_dump.go @@ -6,6 +6,10 @@ type DataDumpViewModel struct { TotalPages int `json:"total_pages"` } +type DataDumpResultErrorModel struct { + Error string `json:"error"` +} + type DataDumpResultViewModel struct { Data *DataDumpData `json:"data"` } diff --git a/services/imports/wakatime_dump.go b/services/imports/wakatime_dump.go index 36a9a0e..046ce39 100644 --- a/services/imports/wakatime_dump.go +++ b/services/imports/wakatime_dump.go @@ -6,6 +6,9 @@ import ( "encoding/json" "errors" "fmt" + "net/http" + "time" + "github.com/duke-git/lancet/v2/slice" "github.com/emvi/logbuch" "github.com/muety/artifex/v2" @@ -13,8 +16,6 @@ import ( "github.com/muety/wakapi/models" wakatime "github.com/muety/wakapi/models/compat/wakatime/v1" "github.com/muety/wakapi/utils" - "net/http" - "time" ) // data example: https://github.com/muety/wakapi/issues/323#issuecomment-1627467052 @@ -37,22 +38,52 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT out := make(chan *models.Heartbeat) logbuch.Info("running wakatime dump import for user '%s'", user.ID) + dump_exist := false url := config.WakatimeApiUrl + config.WakatimeApiDataDumpUrl // this importer only works with wakatime currently, so no point in using user's custom wakatime api url req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(`{ "type": "heartbeats", "email_when_finished": false }`))) res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) if err != nil { - return nil, err + if res.StatusCode == 400 { + var datadumpError wakatime.DataDumpResultErrorModel + if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil { + return nil, err + } + if datadumpError.Error == "Wait for your current export to expire before creating another."{ + dump_exist = true + } + } else { + return nil, err + } } defer res.Body.Close() var datadumpData wakatime.DataDumpResultViewModel - if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil { - return nil, err + if !dump_exist { + if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil { + return nil, err + } } var readyPollTimer *artifex.DispatchTicker // callbacks + getLatestDump := func(user *models.User) (bool, *wakatime.DataDumpData, error) { + req, _ := http.NewRequest(http.MethodGet, url, nil) + res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) + if err != nil { + return false, nil, err + } + + var datadumpData wakatime.DataDumpViewModel + if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil { + return false, nil, err + } + + dump := datadumpData.Data[0] + + return dump.Status == "Completed", dump, nil + } + checkDumpReady := func(dumpId string, user *models.User) (bool, *wakatime.DataDumpData, error) { req, _ := http.NewRequest(http.MethodGet, url, nil) res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) @@ -131,12 +162,21 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT // start polling for dump to be ready readyPollTimer, err = w.queue.DispatchEvery(func() { u := *user - ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u) - logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete) - if err != nil { - onDumpFailed(err, &u) - } else if ok { - onDumpReady(dump, &u, out) + if dump_exist { + ok, dump, err := getLatestDump(&u) + if err != nil { + onDumpFailed(err, &u) + } else if ok { + onDumpReady(dump, &u, out) + } + } else { + ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u) + logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete) + if err != nil { + onDumpFailed(err, &u) + } else if ok { + onDumpReady(dump, &u, out) + } } }, 10*time.Second) From 583ddcab7a932d51aba27783e05785fc022f39e9 Mon Sep 17 00:00:00 2001 From: Edward <73746306+WangEdward@users.noreply.github.com> Date: Fri, 14 Jul 2023 00:33:55 +0800 Subject: [PATCH 2/4] refactor: remove repeated code in readyPollTimer --- services/imports/wakatime_dump.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/services/imports/wakatime_dump.go b/services/imports/wakatime_dump.go index 046ce39..1524338 100644 --- a/services/imports/wakatime_dump.go +++ b/services/imports/wakatime_dump.go @@ -48,7 +48,7 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil { return nil, err } - if datadumpError.Error == "Wait for your current export to expire before creating another."{ + if datadumpError.Error == "Wait for your current export to expire before creating another." { dump_exist = true } } else { @@ -162,21 +162,19 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT // start polling for dump to be ready readyPollTimer, err = w.queue.DispatchEvery(func() { u := *user + var ok bool + var dump *wakatime.DataDumpData + var err error if dump_exist { - ok, dump, err := getLatestDump(&u) - if err != nil { - onDumpFailed(err, &u) - } else if ok { - onDumpReady(dump, &u, out) - } + ok, dump, err = getLatestDump(&u) } else { - ok, dump, err := checkDumpReady(datadumpData.Data.Id, &u) - logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", datadumpData.Data.Id, u.ID, dump.PercentComplete) - if err != nil { - onDumpFailed(err, &u) - } else if ok { - onDumpReady(dump, &u, out) - } + ok, dump, err = checkDumpReady(datadumpData.Data.Id, &u) + } + logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete) + if err != nil { + onDumpFailed(err, &u) + } else if ok { + onDumpReady(dump, &u, out) } }, 10*time.Second) From 56de2757816be64818763bc58b560fceabaa1bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Thu, 13 Jul 2023 20:48:56 +0200 Subject: [PATCH 3/4] chore: simplify import checks fix: minor fixes --- services/imports/wakatime_dump.go | 71 +++++++++---------------------- 1 file changed, 19 insertions(+), 52 deletions(-) diff --git a/services/imports/wakatime_dump.go b/services/imports/wakatime_dump.go index 1524338..ccef769 100644 --- a/services/imports/wakatime_dump.go +++ b/services/imports/wakatime_dump.go @@ -6,16 +6,15 @@ import ( "encoding/json" "errors" "fmt" + "github.com/muety/wakapi/utils" "net/http" "time" - "github.com/duke-git/lancet/v2/slice" "github.com/emvi/logbuch" "github.com/muety/artifex/v2" "github.com/muety/wakapi/config" "github.com/muety/wakapi/models" wakatime "github.com/muety/wakapi/models/compat/wakatime/v1" - "github.com/muety/wakapi/utils" ) // data example: https://github.com/muety/wakapi/issues/323#issuecomment-1627467052 @@ -38,36 +37,31 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT out := make(chan *models.Heartbeat) logbuch.Info("running wakatime dump import for user '%s'", user.ID) - dump_exist := false url := config.WakatimeApiUrl + config.WakatimeApiDataDumpUrl // this importer only works with wakatime currently, so no point in using user's custom wakatime api url req, _ := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(`{ "type": "heartbeats", "email_when_finished": false }`))) res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) - if err != nil { - if res.StatusCode == 400 { - var datadumpError wakatime.DataDumpResultErrorModel - if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil { - return nil, err - } - if datadumpError.Error == "Wait for your current export to expire before creating another." { - dump_exist = true - } + + if err != nil && res.StatusCode == http.StatusBadRequest { + var datadumpError wakatime.DataDumpResultErrorModel + if err := json.NewDecoder(res.Body).Decode(&datadumpError); err != nil { + return nil, err + } + // in case of this error message, a dump had already been requested before and can simply be downloaded now + // -> just keep going as usual (kick off poll loop), otherwise yield error + if datadumpError.Error == "Wait for your current export to expire before creating another." { + logbuch.Info("failed to request new dump, because other non-expired dump already existing, using that one") } else { return nil, err } + } else if err != nil { + return nil, err } defer res.Body.Close() - var datadumpData wakatime.DataDumpResultViewModel - if !dump_exist { - if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil { - return nil, err - } - } - var readyPollTimer *artifex.DispatchTicker // callbacks - getLatestDump := func(user *models.User) (bool, *wakatime.DataDumpData, error) { + checkDumpAvailable := func(user *models.User) (bool, *wakatime.DataDumpData, error) { req, _ := http.NewRequest(http.MethodGet, url, nil) res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) if err != nil { @@ -79,31 +73,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT return false, nil, err } - dump := datadumpData.Data[0] - - return dump.Status == "Completed", dump, nil - } - - checkDumpReady := func(dumpId string, user *models.User) (bool, *wakatime.DataDumpData, error) { - req, _ := http.NewRequest(http.MethodGet, url, nil) - res, err := utils.RaiseForStatus(w.httpClient.Do(w.withHeaders(req))) - if err != nil { - return false, nil, err + if len(datadumpData.Data) < 1 { + return false, nil, errors.New("no dumps available") } - var datadumpData wakatime.DataDumpViewModel - if err := json.NewDecoder(res.Body).Decode(&datadumpData); err != nil { - return false, nil, err - } - - dump, ok := slice.FindBy[*wakatime.DataDumpData](datadumpData.Data, func(i int, item *wakatime.DataDumpData) bool { - return item.Id == dumpId - }) - if !ok { - return false, nil, errors.New(fmt.Sprintf("data dump with id '%s' for user '%s' not found", dumpId, user.ID)) - } - - return dump.Status == "Completed", dump, nil + return datadumpData.Data[0].Status == "Completed", datadumpData.Data[0], nil } onDumpFailed := func(err error, user *models.User) { @@ -162,18 +136,11 @@ func (w *WakatimeDumpImporter) Import(user *models.User, minFrom time.Time, maxT // start polling for dump to be ready readyPollTimer, err = w.queue.DispatchEvery(func() { u := *user - var ok bool - var dump *wakatime.DataDumpData - var err error - if dump_exist { - ok, dump, err = getLatestDump(&u) - } else { - ok, dump, err = checkDumpReady(datadumpData.Data.Id, &u) - } - logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete) + ok, dump, err := checkDumpAvailable(&u) if err != nil { onDumpFailed(err, &u) } else if ok { + logbuch.Info("waiting for data dump '%s' for user '%s' to become downloadable (%.2f percent complete)", dump.Id, u.ID, dump.PercentComplete) onDumpReady(dump, &u, out) } }, 10*time.Second) From bc2d05bd85df5595b0088ae4fd1ac3a043750654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Fri, 14 Jul 2023 08:50:23 +0200 Subject: [PATCH 4/4] ci: skip multi-platform build step on pushes and prs [skip ci] --- .github/workflows/ci.yml | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 838e7c1..7ca4bc0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,33 +75,6 @@ jobs: with: sarif_file: mapi.sarif - build: - name: 'Build (Win, Linux, Mac)' - - strategy: - matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] - - runs-on: ${{ matrix.platform }} - env: - CGO_ENABLED: 0 - - steps: - - name: Set up Go 1.x - uses: actions/setup-go@v3 - with: - go-version: ^1.20 - id: go - - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - - - name: Get dependencies - run: go get - - - name: Build - run: go build -v . - migration: name: Migration tests runs-on: ubuntu-latest