refactor: use job queue for data imports

config/jobqueue.go

@@ -15,6 +15,7 @@ const (
 	QueueDefault    = "wakapi.default"
 	QueueProcessing = "wakapi.processing"
 	QueueReports    = "wakapi.reports"
+	QueueImports    = "wakapi.imports"
 )
 
 type JobQueueMetrics struct {
@@ -29,6 +30,7 @@ func init() {
 	InitQueue(QueueDefault, 1)
 	InitQueue(QueueProcessing, int(math.Ceil(float64(runtime.NumCPU())/2.0)))
 	InitQueue(QueueReports, 1)
+	InitQueue(QueueImports, 1)
 }
 
 func InitQueue(name string, workers int) error {

services/imports/wakatime.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"github.com/duke-git/lancet/v2/datetime"
+	"github.com/muety/artifex"
 	"github.com/muety/wakapi/utils"
 	"net/http"
 	"strings"
@@ -32,19 +33,23 @@ const (
 type WakatimeHeartbeatImporter struct {
 	ApiKey     string
 	httpClient *http.Client
+	queue      *artifex.Dispatcher
 }
 
 func NewWakatimeHeartbeatImporter(apiKey string) *WakatimeHeartbeatImporter {
 	return &WakatimeHeartbeatImporter{
 		ApiKey:     apiKey,
 		httpClient: &http.Client{Timeout: 10 * time.Second},
+		queue:      config.GetQueue(config.QueueImports),
 	}
 }
 
 func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time, maxTo time.Time) <-chan *models.Heartbeat {
 	out := make(chan *models.Heartbeat)
 
-	go func(user *models.User, out chan *models.Heartbeat) {
+	process := func(user *models.User, minFrom time.Time, maxTo time.Time, out chan *models.Heartbeat) {
+		logbuch.Info("running wakatime import for user '%s'", user.ID)
+
 		baseUrl := user.WakaTimeURL(config.WakatimeApiUrl)
 
 		startDate, endDate, err := w.fetchRange(baseUrl)
@@ -109,7 +114,14 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time,
 				}
 			}(d)
 		}
-	}(user, out)
+	}
+
+	logbuch.Info("scheduling wakatime import for user '%s'", user.ID)
+	if err := w.queue.Dispatch(func() {
+		process(user, minFrom, maxTo, out)
+	}); err != nil {
+		config.Log().Error("failed to dispatch wakatime import job for user '%s', %v", user.ID, err)
+	}
 
 	return out
 }