mirror/wakapi
1
0
Fork 0
mirror of https://github.com/muety/wakapi.git synced 2023-08-10 21:12:56 +03:00
Code Issues Projects Releases Wiki Activity

feat: add ability to regenerate summaries

Browse Source 
fix: database cascade settings
chore: debug log mode for gorm queries is back
This commit is contained in:
Ferdinand Mütsch 2020-11-06 17:09:41 +01:00
parent e6a04cc76d
commit e269b37b0e
14 changed files with 194 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
main.go
View File

@ -63,6 +63,9 @@ func main() {
db.Raw("PRAGMA foreign_keys = ON;")
}
if config.IsDev() {
db = db.Debug()
}
sqlDb, _ := db.DB()
sqlDb.SetMaxIdleConns(int(config.Db.MaxConn))
sqlDb.SetMaxOpenConns(int(config.Db.MaxConn))
@ -103,7 +106,7 @@ func main() {
summaryHandler := routes.NewSummaryHandler(summaryService)
healthHandler := routes.NewHealthHandler(db)
heartbeatHandler := routes.NewHeartbeatHandler(heartbeatService, languageMappingService)
settingsHandler := routes.NewSettingsHandler(userService, languageMappingService)
settingsHandler := routes.NewSettingsHandler(userService, summaryService, aggregationService, languageMappingService)
publicHandler := routes.NewIndexHandler(userService, keyValueService)
wakatimeV1AllHandler := wtV1Routes.NewAllTimeHandler(summaryService)
wakatimeV1SummariesHandler := wtV1Routes.NewSummariesHandler(summaryService)
@ -148,10 +151,11 @@ func main() {
// Settings Routes
settingsRouter.Methods(http.MethodGet).HandlerFunc(settingsHandler.GetIndex)
settingsRouter.Path("/credentials").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCredentials)
settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostCreateLanguageMapping)
settingsRouter.Path("/language_mappings").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostLanguageMapping)
settingsRouter.Path("/language_mappings/delete").Methods(http.MethodPost).HandlerFunc(settingsHandler.DeleteLanguageMapping)
settingsRouter.Path("/reset").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostResetApiKey)
settingsRouter.Path("/badges").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostToggleBadges)
settingsRouter.Path("/regenerate").Methods(http.MethodPost).HandlerFunc(settingsHandler.PostRegenerateSummaries)
// API Routes
apiRouter.Path("/heartbeat").Methods(http.MethodPost).HandlerFunc(heartbeatHandler.ApiPost)

65
migrations/common/custom_pre.go
View File

@ -30,6 +30,71 @@ func init() {
},
name: "rename language mappings table",
},
{
f: func(db *gorm.DB, cfg *config.Config) error {
// drop all already existing foreign key constraints
// afterwards let them be re-created by auto migrate with the newly introduced cascade settings,
migrator := db.Migrator()
const lookupKey = "20201106-migration_cascade_constraints"
if cfg.Db.Dialect == "sqlite3" {
// https://stackoverflow.com/a/1884893/3112139
// unfortunately, we can't migrate existing sqlite databases to the newly introduced cascade settings
// things like deleting all summaries won't work in those cases unless an entirely new db is created
log.Println("not attempting to drop and regenerate constraints on sqlite")
return nil
}
if !migrator.HasTable(&models.KeyStringValue{}) {
log.Println("key-value table not yet existing")
return nil
}
condition := "key = ?"
if cfg.Db.Dialect == "mysql" {
condition = "`key` = ?"
}
lookupResult := db.Where(condition, lookupKey).First(&models.KeyStringValue{})
if lookupResult.Error == nil && lookupResult.RowsAffected > 0 {
log.Println("no need to migrate cascade constraints")
return nil
}
// SELECT * FROM INFORMATION_SCHEMA.table_constraints;
constraints := map[string]interface{}{
"fk_summaries_editors": &models.SummaryItem{},
"fk_summaries_languages": &models.SummaryItem{},
"fk_summaries_machines": &models.SummaryItem{},
"fk_summaries_operating_systems": &models.SummaryItem{},
"fk_summaries_projects": &models.SummaryItem{},
"fk_summary_items_summary": &models.SummaryItem{},
"fk_summaries_user": &models.Summary{},
"fk_language_mappings_user": &models.LanguageMapping{},
"fk_heartbeats_user": &models.Heartbeat{},
"fk_aliases_user": &models.Alias{},
}
for name, table := range constraints {
if migrator.HasConstraint(table, name) {
log.Printf("dropping constraint '%s'", name)
if err := migrator.DropConstraint(table, name); err != nil {
return err
}
}
}
if err := db.Create(&models.KeyStringValue{
Key: lookupKey,
Value: "done",
}).Error; err != nil {
return err
}
return nil
},
name: "add cascade constraints",
},
}
}

2
models/alias.go
View File

@ -2,7 +2,7 @@ package models
type Alias struct {
ID uint `gorm:"primary_key"`
Type uint8 `gorm:"not null; index:idx_alias_type_key"`
Type uint8 `gorm:"not null; index:idx_alias_type_key; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
User *User `json:"-" gorm:"not null"`
UserID string `gorm:"not null; index:idx_alias_user"`
Key string `gorm:"not null; index:idx_alias_type_key"`

2
models/heartbeat.go
View File

@ -7,7 +7,7 @@ import (
type Heartbeat struct {
ID uint `gorm:"primary_key"`
User *User `json:"-" gorm:"not null"`
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;"`
UserID string `json:"-" gorm:"not null; index:idx_time_user"`
Entity string `json:"entity" gorm:"not null; index:idx_entity"`
Type string `json:"type"`

2
models/language_mapping.go
View File

@ -2,7 +2,7 @@ package models
type LanguageMapping struct {
ID uint `json:"id" gorm:"primary_key"`
User *User `json:"-" gorm:"not null"`
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
UserID string `json:"-" gorm:"not null; index:idx_language_mapping_user; uniqueIndex:idx_language_mapping_composite"`
Extension string `json:"extension" gorm:"uniqueIndex:idx_language_mapping_composite; type:varchar(16)"`
Language string `json:"language" gorm:"type:varchar(64)"`

14
models/summary.go
View File

@ -35,20 +35,20 @@ const UnknownSummaryKey = "unknown"
type Summary struct {
ID uint `json:"-" gorm:"primary_key"`
User *User `json:"-" gorm:"not null"`
User *User `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
UserID string `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
FromTime CustomTime `json:"from" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
ToTime CustomTime `json:"to" gorm:"not null; type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time_summary_user"`
Projects []*SummaryItem `json:"projects"`
Languages []*SummaryItem `json:"languages"`
Editors []*SummaryItem `json:"editors"`
OperatingSystems []*SummaryItem `json:"operating_systems"`
Machines []*SummaryItem `json:"machines"`
Projects []*SummaryItem `json:"projects" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Languages []*SummaryItem `json:"languages" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Editors []*SummaryItem `json:"editors" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
OperatingSystems []*SummaryItem `json:"operating_systems" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Machines []*SummaryItem `json:"machines" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
}
type SummaryItem struct {
ID uint `json:"-" gorm:"primary_key"`
Summary *Summary `json:"-" gorm:"not null"`
Summary *Summary `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
SummaryID uint `json:"-"`
Type uint8 `json:"-"`
Key string `json:"key"`

9
repositories/summary.go
View File

@ -51,3 +51,12 @@ func (r *SummaryRepository) GetLatestByUser() ([]*models.Summary, error) {
}
return summaries, nil
}
func (r *SummaryRepository) DeleteByUser(userId string) error {
if err := r.db.
Where("user_id = ?", userId).
Delete(models.Summary{}).Error; err != nil {
return err
}
return nil
}

33
routes/settings.go
View File

@ -7,6 +7,7 @@ import (
"github.com/muety/wakapi/models"
"github.com/muety/wakapi/services"
"github.com/muety/wakapi/utils"
"log"
"net/http"
"net/url"
"strconv"
@ -15,14 +16,18 @@ import (
type SettingsHandler struct {
config *conf.Config
userSrvc *services.UserService
summarySrvc *services.SummaryService
aggregationSrvc *services.AggregationService
languageMappingSrvc *services.LanguageMappingService
}
var credentialsDecoder = schema.NewDecoder()
func NewSettingsHandler(userService *services.UserService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
func NewSettingsHandler(userService *services.UserService, summaryService *services.SummaryService, aggregationService *services.AggregationService, languageMappingService *services.LanguageMappingService) *SettingsHandler {
return &SettingsHandler{
config: conf.Get(),
summarySrvc: summaryService,
aggregationSrvc: aggregationService,
languageMappingSrvc: languageMappingService,
userSrvc: userService,
}
@ -133,7 +138,7 @@ func (h *SettingsHandler) DeleteLanguageMapping(w http.ResponseWriter, r *http.R
http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("mapping deleted successfully")), http.StatusFound)
}
func (h *SettingsHandler) PostCreateLanguageMapping(w http.ResponseWriter, r *http.Request) {
func (h *SettingsHandler) PostLanguageMapping(w http.ResponseWriter, r *http.Request) {
if h.config.IsDev() {
loadTemplates()
}
@ -180,7 +185,6 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
}
user := r.Context().Value(models.UserKey).(*models.User)
if _, err := h.userSrvc.ToggleBadges(user); err != nil {
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("internal server error")), http.StatusFound)
return
@ -188,3 +192,26 @@ func (h *SettingsHandler) PostToggleBadges(w http.ResponseWriter, r *http.Reques
http.Redirect(w, r, fmt.Sprintf("%s/settings", h.config.Server.BasePath), http.StatusFound)
}
func (h *SettingsHandler) PostRegenerateSummaries(w http.ResponseWriter, r *http.Request) {
if h.config.IsDev() {
loadTemplates()
}
user := r.Context().Value(models.UserKey).(*models.User)
log.Printf("clearing summaries for user '%s'\n", user.ID)
if err := h.summarySrvc.DeleteByUser(user.ID); err != nil {
log.Printf("failed to clear summaries: %v\n", err)
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to delete old summaries")), http.StatusFound)
return
}
if err := h.aggregationSrvc.Run(map[string]bool{user.ID: true}); err != nil {
log.Printf("failed to regenerate summaries: %v\n", err)
http.Redirect(w, r, fmt.Sprintf("%s/settings?error=%s", h.config.Server.BasePath, url.QueryEscape("failed to generate aggregations")), http.StatusFound)
return
}
http.Redirect(w, r, fmt.Sprintf("%s/settings?success=%s", h.config.Server.BasePath, url.QueryEscape("summaries are being regenerated – this may take a few second")), http.StatusFound)
}

33
services/aggregation.go
View File

@ -38,10 +38,18 @@ type AggregationJob struct {
// Schedule a job to (re-)generate summaries every day shortly after midnight
func (srv *AggregationService) Schedule() {
// Run once initially
if err := srv.Run(nil); err != nil {
log.Fatalf("failed to run aggregation jobs: %v\n", err)
}
gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.Run, nil)
<-gocron.Start()
}
func (srv *AggregationService) Run(userIds map[string]bool) error {
jobs := make(chan *AggregationJob)
summaries := make(chan *models.Summary)
defer close(jobs)
defer close(summaries)
for i := 0; i < runtime.NumCPU(); i++ {
go srv.summaryWorker(jobs, summaries)
@ -51,11 +59,7 @@ func (srv *AggregationService) Schedule() {
go srv.persistWorker(summaries)
}
// Run once initially
srv.trigger(jobs)
gocron.Every(1).Day().At(srv.config.App.AggregationTime).Do(srv.trigger, jobs)
<-gocron.Start()
return srv.trigger(jobs, userIds)
}
func (srv *AggregationService) summaryWorker(jobs <-chan *AggregationJob, summaries chan<- *models.Summary) {
@ -77,13 +81,22 @@ func (srv *AggregationService) persistWorker(summaries <-chan *models.Summary) {
}
}
func (srv *AggregationService) trigger(jobs chan<- *AggregationJob) error {
func (srv *AggregationService) trigger(jobs chan<- *AggregationJob, userIds map[string]bool) error {
log.Println("Generating summaries.")
users, err := srv.userService.GetAll()
if err != nil {
var users []*models.User
if allUsers, err := srv.userService.GetAll(); err != nil {
log.Println(err)
return err
} else if userIds != nil && len(userIds) > 0 {
users = make([]*models.User, len(userIds))
for i, u := range allUsers {
if yes, ok := userIds[u.ID]; yes && ok {
users[i] = u
}
}
} else {
users = allUsers
}
latestSummaries, err := srv.summaryService.GetLatestByUser()

4
services/summary.go
View File

@ -225,6 +225,10 @@ func (srv *SummaryService) GetLatestByUser() ([]*models.Summary, error) {
return srv.repository.GetLatestByUser()
}
func (srv *SummaryService) DeleteByUser(userId string) error {
return srv.repository.DeleteByUser(userId)
}
func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryType uint8, user *models.User, c chan models.SummaryItemContainer) {
durations := make(map[string]time.Duration)

2
version.txt
View File

@ -1 +1 @@
1.14.1
1.15.0

42
views/settings.tpl.html
View File

@ -48,7 +48,7 @@
</form>
</div>
<div class="w-full mt-4 mb-8 pb-8">
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
Reset API Key
</div>
@ -66,7 +66,7 @@
</form>
</div>
<div class="w-full mt-4 mb-8 pb-8">
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
Language Mappings
</div>
@ -119,7 +119,7 @@
</div>
<div class="w-full mt-4 mb-8 pb-8">
<div class="w-full mt-4 mb-8 pb-8 border-b border-gray-700">
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
Badges
</div>
@ -169,6 +169,34 @@
</div>
</form>
</div>
<div class="w-full mt-4 mb-8 pb-8">
<div class="font-semibold text-lg text-white m-0 border-b-2 border-green-700 inline-block">
⚠️ Danger Zone
</div>
<div class="mt-10 text-gray-300 text-sm">
<h3 class="font-semibold text-md mb-4 border-b border-green-700 inline-block">
Regenerate summaries
</h3>
<p>
Wakapi improves its efficiency and speed by automatically aggregating individual heartbeats to summaries on a per-day basis.
That is, historic summaries, i.e. such from past days, are generated once and only fetched from the database in a static fashion afterwards, unless you pass <span class="font-mono font-normal bg-gray-900 p-1 rounded whitespace-no-wrap">&recompute=true</span> with your request.
</p>
<p class="mt-2">
If, for some reason, these aggregated summaries are faulty or preconditions have change (e.g. you modified language mappings retrospectively), you may want to re-generate them from raw heartbeats.
</p>
<p class="mt-2">
<strong>Note:</strong> Only run this action if you know what you are doing. Data might be lost is case heartbeats were deleted after the respective summaries had been generated.
</p>
</div>
<div class="mt-10 flex justify-center">
<form action="settings/regenerate" method="post" id="form-regenerate-summaries">
<button type="button" class="py-1 px-3 rounded bg-red-500 hover:bg-red-600 text-white text-sm" id="btn-regenerate-summaries">
Clear & Regenerate
</button>
</form>
</div>
</div>
</div>
</main>
@ -182,6 +210,14 @@
e.innerHTML = e.innerHTML.replace('%s', baseUrl)
e.classList.remove('hidden')
})
const btnRegenerate = document.querySelector("#btn-regenerate-summaries")
const formRegenerate = document.querySelector('#form-regenerate-summaries')
btnRegenerate.addEventListener('click', () => {
if (confirm('Are you sure?')) {
formRegenerate.submit()
}
})
</script>
{{ template "footer.tpl.html" . }}