feat: allow to configure custom api url for relay and import (resolve #105)

This commit is contained in:
Ferdinand Mütsch 2022-01-21 12:35:05 +01:00
parent fce3a3ea20
commit 7159df30c2
15 changed files with 880 additions and 742 deletions

View File

@ -4,6 +4,7 @@ import (
"encoding/json"
"flag"
"fmt"
uuid "github.com/satori/go.uuid"
"io/ioutil"
"net/http"
"os"
@ -143,6 +144,7 @@ type Config struct {
Env string `default:"dev" env:"ENVIRONMENT"`
Version string `yaml:"-"`
QuickStart bool `yaml:"quick_start" env:"WAKAPI_QUICK_START"`
InstanceId string `yaml:"-"` // only temporary, changes between runs
App appConfig
Security securityConfig
Db dbConfig
@ -355,6 +357,7 @@ func Load(version string) *Config {
env = config.Env
config.Version = strings.TrimSpace(version)
config.InstanceId = uuid.NewV4().String()
config.App.Colors = readColors()
config.Db.Dialect = resolveDbDialect(config.Db.Type)
config.Security.SecureCookie = securecookie.New(

File diff suppressed because it is too large Load Diff

View File

@ -3,12 +3,15 @@ package relay
import (
"bytes"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"github.com/emvi/logbuch"
"github.com/leandro-lugaresi/hub"
"github.com/muety/wakapi/config"
"github.com/muety/wakapi/middlewares"
"github.com/muety/wakapi/models"
routeutils "github.com/muety/wakapi/routes/utils"
"github.com/patrickmn/go-cache"
"io"
"io/ioutil"
@ -18,9 +21,10 @@ import (
const maxFailuresPerDay = 100
/* Middleware to conditionally relay heartbeats to Wakatime */
// WakatimeRelayMiddleware is a middleware to conditionally relay heartbeats to Wakatime (and other compatible services)
type WakatimeRelayMiddleware struct {
httpClient *http.Client
hashCache *cache.Cache
failureCache *cache.Cache
eventBus *hub.Hub
}
@ -30,6 +34,7 @@ func NewWakatimeRelayMiddleware() *WakatimeRelayMiddleware {
httpClient: &http.Client{
Timeout: 10 * time.Second,
},
hashCache: cache.New(10*time.Minute, 10*time.Minute),
failureCache: cache.New(24*time.Hour, 1*time.Hour),
eventBus: config.EventBus(),
}
@ -44,7 +49,10 @@ func (m *WakatimeRelayMiddleware) Handler(h http.Handler) http.Handler {
func (m *WakatimeRelayMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
defer next(w, r)
if r.Method != http.MethodPost {
ownInstanceId := config.Get().InstanceId
originInstanceId := r.Header.Get("X-Origin-Instance")
if r.Method != http.MethodPost || originInstanceId == ownInstanceId {
return
}
@ -53,10 +61,22 @@ func (m *WakatimeRelayMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Reque
return
}
err := m.filterByCache(r)
if err != nil {
logbuch.Warn("%v", err)
return
}
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
// prevent cycles
downstreamInstanceId := ownInstanceId
if originInstanceId != "" {
downstreamInstanceId = originInstanceId
}
headers := http.Header{
"X-Machine-Name": r.Header.Values("X-Machine-Name"),
"Content-Type": r.Header.Values("Content-Type"),
@ -65,14 +85,17 @@ func (m *WakatimeRelayMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Reque
"X-Origin": []string{
fmt.Sprintf("wakapi v%s", config.Get().Version),
},
"X-Origin-Instance": []string{downstreamInstanceId},
"Authorization": []string{
fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(user.WakatimeApiKey))),
},
}
url := user.WakaTimeURL(config.WakatimeApiUrl) + config.WakatimeApiHeartbeatsBulkUrl
go m.send(
http.MethodPost,
config.WakatimeApiUrl+config.WakatimeApiHeartbeatsBulkUrl,
url,
bytes.NewReader(body),
headers,
user,
@ -115,3 +138,53 @@ func (m *WakatimeRelayMiddleware) send(method, url string, body io.Reader, heade
}
}
}
// filterByCache takes an HTTP request, tries to parse the body contents as heartbeats, checks against a local cache for whether a heartbeat has already been relayed before according to its hash and in-place filters these from the request's raw json body.
// This method operates on the raw body data (interface{}), because serialization of models.Heartbeat is not necessarily identical to what the CLI has actually sent.
// Purpose of this mechanism is mainly to prevent cyclic relays / loops.
// Caution: this method does in-place changes to the request.
func (m *WakatimeRelayMiddleware) filterByCache(r *http.Request) error {
heartbeats, err := routeutils.ParseHeartbeats(r)
if err != nil {
return err
}
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
var rawData interface{}
if err := json.NewDecoder(ioutil.NopCloser(bytes.NewBuffer(body))).Decode(&rawData); err != nil {
return err
}
newData := make([]interface{}, 0, len(heartbeats))
for i, hb := range heartbeats {
hb = hb.Hashed()
// we didn't see this particular heartbeat before
if _, found := m.hashCache.Get(hb.Hash); !found {
m.hashCache.SetDefault(hb.Hash, true)
newData = append(newData, rawData.([]interface{})[i])
continue
}
}
if len(newData) == 0 {
return errors.New("no new heartbeats to relay")
}
if len(newData) != len(heartbeats) {
user := middlewares.GetPrincipal(r)
logbuch.Warn("only relaying %d of %d heartbeats for user %s", len(newData), len(heartbeats), user.ID)
}
buf := bytes.Buffer{}
if err := json.NewEncoder(&buf).Encode(newData); err != nil {
return err
}
r.Body = ioutil.NopCloser(&buf)
return nil
}

View File

@ -74,8 +74,8 @@ func (m *UserServiceMock) ToggleBadges(user *models.User) (*models.User, error)
return args.Get(0).(*models.User), args.Error(1)
}
func (m *UserServiceMock) SetWakatimeApiKey(user *models.User, s string) (*models.User, error) {
args := m.Called(user, s)
func (m *UserServiceMock) SetWakatimeApiCredentials(user *models.User, s1, s2 string) (*models.User, error) {
args := m.Called(user, s1, s2)
return args.Get(0).(*models.User), args.Error(1)
}

View File

@ -29,7 +29,8 @@ type User struct {
ShareLabels bool `json:"-" gorm:"default:false; type:bool"`
IsAdmin bool `json:"-" gorm:"default:false; type:bool"`
HasData bool `json:"-" gorm:"default:false; type:bool"`
WakatimeApiKey string `json:"-"`
WakatimeApiKey string `json:"-"` // for relay middleware and imports
WakatimeApiUrl string `json:"-"` // for relay middleware and imports
ResetToken string `json:"-"`
ReportsWeekly bool `json:"-" gorm:"default:false; type:bool"`
}
@ -109,6 +110,14 @@ func (u *User) AvatarURL(urlTemplate string) string {
return urlTemplate
}
// WakaTimeURL returns the user's effective WakaTime URL, i.e. a custom one (which could also point to another Wakapi instance) or fallback if not specified otherwise.
func (u *User) WakaTimeURL(fallback string) string {
if u.WakatimeApiUrl != "" {
return strings.TrimSuffix(u.WakatimeApiUrl, "/")
}
return fallback
}
func (c *CredentialsReset) IsValid() bool {
return ValidatePassword(c.PasswordNew) &&
c.PasswordNew == c.PasswordRepeat

View File

@ -152,6 +152,7 @@ func (r *UserRepository) Update(user *models.User) (*models.User, error) {
"share_machines": user.ShareMachines,
"share_labels": user.ShareLabels,
"wakatime_api_key": user.WakatimeApiKey,
"wakatime_api_url": user.WakatimeApiUrl,
"has_data": user.HasData,
"reset_token": user.ResetToken,
"location": user.Location,

View File

@ -1,9 +1,6 @@
package api
import (
"bytes"
"encoding/json"
"io/ioutil"
"net/http"
"github.com/gorilla/mux"
@ -69,15 +66,12 @@ func (h *HeartbeatApiHandler) Post(w http.ResponseWriter, r *http.Request) {
}
var heartbeats []*models.Heartbeat
heartbeats, err = h.tryParseBulk(r)
heartbeats, err = routeutils.ParseHeartbeats(r)
if err != nil {
heartbeats, err = h.tryParseSingle(r)
if err != nil {
conf.Log().Request(r).Error(err.Error())
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
conf.Log().Request(r).Error(err.Error())
w.WriteHeader(http.StatusBadRequest)
w.Write([]byte(err.Error()))
return
}
userAgent := r.Header.Get("User-Agent")
@ -123,36 +117,6 @@ func (h *HeartbeatApiHandler) Post(w http.ResponseWriter, r *http.Request) {
utils.RespondJSON(w, r, http.StatusCreated, constructSuccessResponse(len(heartbeats)))
}
func (h *HeartbeatApiHandler) tryParseBulk(r *http.Request) ([]*models.Heartbeat, error) {
var heartbeats []*models.Heartbeat
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
dec := json.NewDecoder(ioutil.NopCloser(bytes.NewBuffer(body)))
if err := dec.Decode(&heartbeats); err != nil {
return nil, err
}
return heartbeats, nil
}
func (h *HeartbeatApiHandler) tryParseSingle(r *http.Request) ([]*models.Heartbeat, error) {
var heartbeat models.Heartbeat
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
dec := json.NewDecoder(ioutil.NopCloser(bytes.NewBuffer(body)))
if err := dec.Decode(&heartbeat); err != nil {
return nil, err
}
return []*models.Heartbeat{&heartbeat}, nil
}
// construct weird response format (see https://github.com/wakatime/wakatime/blob/2e636d389bf5da4e998e05d5285a96ce2c181e3d/wakatime/api.py#L288)
// to make the cli consider all heartbeats to having been successfully saved
// response looks like: { "responses": [ [ null, 201 ], ... ] }

View File

@ -56,6 +56,9 @@ func DefaultTemplateFuncs() template.FuncMap {
"avatarUrlTemplate": func() string {
return config.Get().App.AvatarURLTemplate
},
"defaultWakatimeUrl": func() string {
return config.WakatimeApiUrl
},
}
}

View File

@ -431,13 +431,17 @@ func (h *SettingsHandler) actionSetWakatimeApiKey(w http.ResponseWriter, r *http
user := middlewares.GetPrincipal(r)
apiKey := r.PostFormValue("api_key")
apiUrl := r.PostFormValue("api_url")
if apiUrl == conf.WakatimeApiUrl || apiKey == "" {
apiUrl = ""
}
// Healthcheck, if a new API key is set, i.e. the feature is activated
if (user.WakatimeApiKey == "" && apiKey != "") && !h.validateWakatimeKey(apiKey) {
if (user.WakatimeApiKey == "" && apiKey != "") && !h.validateWakatimeKey(apiKey, apiUrl) {
return http.StatusBadRequest, "", "failed to connect to WakaTime, API key invalid?"
}
if _, err := h.userSrvc.SetWakatimeApiKey(user, apiKey); err != nil {
if _, err := h.userSrvc.SetWakatimeApiCredentials(user, apiKey, apiUrl); err != nil {
return http.StatusInternalServerError, "", conf.ErrInternalServerError
}
@ -570,7 +574,11 @@ func (h *SettingsHandler) actionDeleteUser(w http.ResponseWriter, r *http.Reques
return -1, "", ""
}
func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
func (h *SettingsHandler) validateWakatimeKey(apiKey string, baseUrl string) bool {
if baseUrl == "" {
baseUrl = conf.WakatimeApiUrl
}
headers := http.Header{
"Accept": []string{"application/json"},
"Authorization": []string{
@ -580,7 +588,7 @@ func (h *SettingsHandler) validateWakatimeKey(apiKey string) bool {
request, err := http.NewRequest(
http.MethodGet,
conf.WakatimeApiUrl+conf.WakatimeApiUserUrl,
baseUrl+conf.WakatimeApiUserUrl,
nil,
)
if err != nil {

View File

@ -0,0 +1,53 @@
package utils
import (
"bytes"
"encoding/json"
"github.com/muety/wakapi/models"
"io/ioutil"
"net/http"
)
func ParseHeartbeats(r *http.Request) ([]*models.Heartbeat, error) {
heartbeats, err := tryParseBulk(r)
if err == nil {
return heartbeats, err
}
heartbeats, err = tryParseSingle(r)
if err == nil {
return heartbeats, err
}
return []*models.Heartbeat{}, err
}
func tryParseBulk(r *http.Request) ([]*models.Heartbeat, error) {
var heartbeats []*models.Heartbeat
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
dec := json.NewDecoder(ioutil.NopCloser(bytes.NewBuffer(body)))
if err := dec.Decode(&heartbeats); err != nil {
return nil, err
}
return heartbeats, nil
}
func tryParseSingle(r *http.Request) ([]*models.Heartbeat, error) {
var heartbeat models.Heartbeat
body, _ := ioutil.ReadAll(r.Body)
r.Body.Close()
r.Body = ioutil.NopCloser(bytes.NewBuffer(body))
dec := json.NewDecoder(ioutil.NopCloser(bytes.NewBuffer(body)))
if err := dec.Decode(&heartbeat); err != nil {
return nil, err
}
return []*models.Heartbeat{&heartbeat}, nil
}

View File

@ -40,7 +40,9 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time,
out := make(chan *models.Heartbeat)
go func(user *models.User, out chan *models.Heartbeat) {
startDate, endDate, err := w.fetchRange()
baseUrl := user.WakaTimeURL(config.WakatimeApiUrl)
startDate, endDate, err := w.fetchRange(baseUrl)
if err != nil {
config.Log().Error("failed to fetch date range while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
@ -53,13 +55,13 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time,
endDate = maxTo
}
userAgents, err := w.fetchUserAgents()
userAgents, err := w.fetchUserAgents(baseUrl)
if err != nil {
config.Log().Error("failed to fetch user agents while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
}
machinesNames, err := w.fetchMachineNames()
machinesNames, err := w.fetchMachineNames(baseUrl)
if err != nil {
config.Log().Error("failed to fetch machine names while importing wakatime heartbeats for user '%s' %v", user.ID, err)
return
@ -82,7 +84,7 @@ func (w *WakatimeHeartbeatImporter) Import(user *models.User, minFrom time.Time,
defer time.Sleep(throttleDelay)
d := day.Format(config.SimpleDateFormat)
heartbeats, err := w.fetchHeartbeats(d)
heartbeats, err := w.fetchHeartbeats(d, baseUrl)
if err != nil {
config.Log().Error("failed to fetch heartbeats for day '%s' and user '%s' &v", d, user.ID, err)
}
@ -107,10 +109,10 @@ func (w *WakatimeHeartbeatImporter) ImportAll(user *models.User) <-chan *models.
// https://wakatime.com/api/v1/users/current/heartbeats?date=2021-02-05
// https://pastr.de/p/b5p4od5s8w0pfntmwoi117jy
func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.HeartbeatEntry, error) {
func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string, baseUrl string) ([]*wakatime.HeartbeatEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiHeartbeatsUrl, nil)
req, err := http.NewRequest(http.MethodGet, baseUrl+config.WakatimeApiHeartbeatsUrl, nil)
if err != nil {
return nil, err
}
@ -136,12 +138,12 @@ func (w *WakatimeHeartbeatImporter) fetchHeartbeats(day string) ([]*wakatime.Hea
// https://wakatime.com/api/v1/users/current/all_time_since_today
// https://pastr.de/p/w8xb4biv575pu32pox7jj2gr
func (w *WakatimeHeartbeatImporter) fetchRange() (time.Time, time.Time, error) {
func (w *WakatimeHeartbeatImporter) fetchRange(baseUrl string) (time.Time, time.Time, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
notime := time.Time{}
req, err := http.NewRequest(http.MethodGet, config.WakatimeApiUrl+config.WakatimeApiAllTimeUrl, nil)
req, err := http.NewRequest(http.MethodGet, baseUrl+config.WakatimeApiAllTimeUrl, nil)
if err != nil {
return notime, notime, err
}
@ -171,13 +173,13 @@ func (w *WakatimeHeartbeatImporter) fetchRange() (time.Time, time.Time, error) {
// https://wakatime.com/api/v1/users/current/user_agents
// https://pastr.de/p/05k5do8q108k94lic4lfl3pc
func (w *WakatimeHeartbeatImporter) fetchUserAgents() (map[string]*wakatime.UserAgentEntry, error) {
func (w *WakatimeHeartbeatImporter) fetchUserAgents(baseUrl string) (map[string]*wakatime.UserAgentEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
userAgents := make(map[string]*wakatime.UserAgentEntry)
for page := 1; ; page++ {
url := fmt.Sprintf("%s%s?page=%d", config.WakatimeApiUrl, config.WakatimeApiUserAgentsUrl, page)
url := fmt.Sprintf("%s%s?page=%d", baseUrl, config.WakatimeApiUserAgentsUrl, page)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err
@ -207,13 +209,13 @@ func (w *WakatimeHeartbeatImporter) fetchUserAgents() (map[string]*wakatime.User
// https://wakatime.com/api/v1/users/current/machine_names
// https://pastr.de/p/v58cv0xrupp3zvyyv8o6973j
func (w *WakatimeHeartbeatImporter) fetchMachineNames() (map[string]*wakatime.MachineEntry, error) {
func (w *WakatimeHeartbeatImporter) fetchMachineNames(baseUrl string) (map[string]*wakatime.MachineEntry, error) {
httpClient := &http.Client{Timeout: 10 * time.Second}
machines := make(map[string]*wakatime.MachineEntry)
for page := 1; ; page++ {
url := fmt.Sprintf("%s%s?page=%d", config.WakatimeApiUrl, config.WakatimeApiMachineNamesUrl, page)
url := fmt.Sprintf("%s%s?page=%d", baseUrl, config.WakatimeApiMachineNamesUrl, page)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, err

View File

@ -107,7 +107,7 @@ type IUserService interface {
Update(*models.User) (*models.User, error)
Delete(*models.User) error
ResetApiKey(*models.User) (*models.User, error)
SetWakatimeApiKey(*models.User, string) (*models.User, error)
SetWakatimeApiCredentials(*models.User, string, string) (*models.User, error)
MigrateMd5Password(*models.User, *models.Login) (*models.User, error)
GenerateResetToken(*models.User) (*models.User, error)
FlushCache()

View File

@ -38,7 +38,7 @@ func NewUserService(mailService IMailService, userRepo repositories.IUserReposit
logbuch.Warn("resetting wakatime api key for user %s, because of too many failures (%d)", user.ID, n)
if _, err := srv.SetWakatimeApiKey(user, ""); err != nil {
if _, err := srv.SetWakatimeApiCredentials(user, "", ""); err != nil {
logbuch.Error("failed to set wakatime api key for user %s", user.ID)
}
@ -154,9 +154,20 @@ func (srv *UserService) ResetApiKey(user *models.User) (*models.User, error) {
return srv.Update(user)
}
func (srv *UserService) SetWakatimeApiKey(user *models.User, apiKey string) (*models.User, error) {
func (srv *UserService) SetWakatimeApiCredentials(user *models.User, apiKey string, apiUrl string) (*models.User, error) {
srv.cache.Flush()
return srv.repository.UpdateField(user, "wakatime_api_key", apiKey)
if apiKey != user.WakatimeApiKey {
if u, err := srv.repository.UpdateField(user, "wakatime_api_key", apiKey); err != nil {
return u, err
}
}
if apiUrl != user.WakatimeApiUrl {
return srv.repository.UpdateField(user, "wakatime_api_url", apiUrl)
}
return user, nil
}
func (srv *UserService) MigrateMd5Password(user *models.User, login *models.Login) (*models.User, error) {

View File

@ -1 +1 @@
2.0.2
2.1.0

View File

@ -489,14 +489,17 @@
<div class="w-full md:w-1/2 mb-4 md:mb-0 inline-block">
<label class="font-semibold text-gray-300" for="select-timezone">WakaTime</label>
<span class="block text-sm text-gray-600">
You can connect Wakapi with the official WakaTime in a way that all heartbeats sent to Wakapi are relayed. This way, you can use both services at the same time. To get started, get <a class="link" href="https://wakatime.com/developers#authentication" rel="noopener noreferrer" target="_blank">your API key</a> and paste it here.<br><br>
You can connect Wakapi with the official WakaTime (or another Wakapi instance, when optionally specifying a custom API URL) in a way that all heartbeats sent to Wakapi are relayed. This way, you can use both services at the same time. To get started, get <a class="link" href="https://wakatime.com/developers#authentication" rel="noopener noreferrer" target="_blank">your API key</a> and paste it here.<br><br>
Please note: When enabling this feature, the operators of this server will, in theory, have unlimited access to your data stored in WakaTime. If you are concerned about your privacy, please do not enable this integration or wait for OAuth 2 authentication (<a
class="link" target="_blank" href="https://github.com/muety/wakapi/issues/94" rel="noopener noreferrer">#94</a>) to be implemented.
</span>
</div>
<div class="w-full md:w-1/2">
<input type="url" name="api_url" id="wakatime_api_url"
class="w-full appearance-none bg-gray-850 text-gray-300 outline-none rounded py-2 px-4 mb-2 {{ if not .User.WakatimeApiKey }}focus:bg-gray-800{{ end }} {{ if .User.WakatimeApiKey }}cursor-not-allowed{{ end }}"
placeholder="{{ defaultWakatimeUrl }}" {{ if .User.WakatimeApiKey }}readonly{{ end }} value="{{ .User.WakaTimeURL "" }}">
<input type="password" name="api_key" id="wakatime_api_key"
class="w-full appearance-none bg-gray-850 text-gray-300 outline-none rounded py-2 px-4 {{ if not .User.WakatimeApiKey }}focus:bg-gray-800{{ end }} {{ if .User.WakatimeApiKey }}cursor-not-allowed{{ end }}"
class="w-full appearance-none bg-gray-850 text-gray-300 outline-none rounded py-2 px-4 mt-2 {{ if not .User.WakatimeApiKey }}focus:bg-gray-800{{ end }} {{ if .User.WakatimeApiKey }}cursor-not-allowed{{ end }}"
placeholder="{{ $placeholderText }}" {{ if .User.WakatimeApiKey }}readonly{{ end }}>
</div>
</div>