fix: generate dummy items for missing types in historic summary data

feat: add base_path environment variable

Dockerfile

@@ -17,8 +17,7 @@ RUN cd /src && go build -o wakapi
 # – WAKAPI_DB_PORT
 # – WAKAPI_DB_NAME
 # – WAKAPI_PASSWORD_SALT
-# – WAKAPI_DEFAULT_USER_NAME
+# – WAKAPI_BASE_PATH
-# – WAKAPI_DEFAULT_USER_PASSWORD
 
 FROM debian
 WORKDIR /app
@@ -30,8 +29,6 @@ ENV WAKAPI_DB_PASSWORD ''
 ENV WAKAPI_DB_HOST ''
 ENV WAKAPI_DB_NAME=/data/wakapi.db
 ENV WAKAPI_PASSWORD_SALT ''
-ENV WAKAPI_DEFAULT_USER_NAME admin
-ENV WAKAPI_DEFAULT_USER_PASSWORD admin
 
 COPY --from=build-env /src/wakapi /app/
 COPY --from=build-env /src/config.ini /app/

README.md

@@ -80,6 +80,7 @@ INSERT INTO aliases (`type`, `user_id`, `key`, `value`) VALUES (0, 'your_usernam
 * Language ~  type **1**
 * Editor ~ type **2**
 * OS ~  type **3**
+* Machine ~  type **4**
 
 **NOTE:** In order for the aliases to take effect for non-live statistics, you would either have to wait 24 hours for the cache to be invalidated or restart Wakapi.
 

main.go

@@ -43,6 +43,11 @@ func main() {
 		log.SetFlags(log.LstdFlags | log.Lshortfile)
 	}
 
+	// Show data loss warning
+	if config.CleanUp {
+		promptAbort("`CLEANUP` is set to `true`, which may cause data loss. Are you sure to continue?", 5)
+	}
+
 	// Connect to database
 	var err error
 	db, err = gorm.Open(config.DbDialect, utils.MakeConnectionString(config))
@@ -172,3 +177,11 @@ func migrateLanguages() {
 		}
 	}
 }
+
+func promptAbort(message string, timeoutSec int) {
+	log.Printf("[WARNING] %s.\nTo abort server startup, press Ctrl+C.\n", message)
+	for i := timeoutSec; i > 0; i-- {
+		log.Printf("Starting in %d seconds ...\n", i)
+		time.Sleep(1 * time.Second)
+	}
+}

migrations/sqlite3/4_machine_column.sql

@@ -0,0 +1,11 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+alter table users
+    add `machine` varchar(255);
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+alter table users
+    drop column `machine`;

models/config.go

@@ -161,7 +161,11 @@ func readConfig() *Config {
 		port = cfg.Section("server").Key("port").MustInt()
 	}
 
+	basePathEnv, basePathEnvExists := os.LookupEnv("WAKAPI_BASE_PATH")
 	basePath := cfg.Section("server").Key("base_path").MustString("/")
+	if basePathEnvExists {
+		basePath = basePathEnv
+	}
 	if strings.HasSuffix(basePath, "/") {
 		basePath = basePath[:len(basePath)-1]
 	}

models/heartbeat.go

@@ -20,6 +20,7 @@ type Heartbeat struct {
 	IsWrite         bool       `json:"is_write"`
 	Editor          string     `json:"editor"`
 	OperatingSystem string     `json:"operating_system"`
+	Machine         string     `json:"machine"`
 	Time            CustomTime `json:"time" gorm:"type:timestamp; default:CURRENT_TIMESTAMP; index:idx_time,idx_time_user"`
 	languageRegex   *regexp.Regexp
 }

models/summary.go

@@ -5,13 +5,16 @@ import (
 )
 
 const (
-	NSummaryTypes   uint8 = 4
+	NSummaryTypes   uint8 = 99
 	SummaryProject  uint8 = 0
 	SummaryLanguage uint8 = 1
 	SummaryEditor   uint8 = 2
 	SummaryOS       uint8 = 3
+	SummaryMachine  uint8 = 4
 )
 
+const UnknownSummaryKey = "unknown"
+
 type Summary struct {
 	ID               uint           `json:"-" gorm:"primary_key"`
 	UserID           string         `json:"user_id" gorm:"not null; index:idx_time_summary_user"`
@@ -21,6 +24,7 @@ type Summary struct {
 	Languages        []*SummaryItem `json:"languages"`
 	Editors          []*SummaryItem `json:"editors"`
 	OperatingSystems []*SummaryItem `json:"operating_systems"`
+	Machines         []*SummaryItem `json:"machines"`
 }
 
 type SummaryItem struct {
@@ -43,3 +47,55 @@ type SummaryViewModel struct {
 	Success        string
 	ApiKey         string
 }
+
+/* Augments the summary in a way that at least one item is present for every type.
+If a summary has zero items for a given type, but one or more for any of the other types,
+the total summary duration can be derived from those and inserted as a dummy-item with key "unknown"
+for the missing type.
+For instance, the machine type was introduced post hoc. Accordingly, no "machine"-information is present in
+the data for old heartbeats and summaries. If a user has two years of data without machine information and
+one day with such, a "machine"-chart plotted from that data will reference a way smaller absolute total amount
+of time than the other ones.
+To avoid having to modify persisted data retrospectively, i.e. inserting a dummy SummaryItem for the new type,
+such is generated dynamically here, considering the "machine" for all old heartbeats "unknown".
+*/
+func (s *Summary) FillUnknown() {
+	types := []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine}
+	missingTypes := make([]uint8, 0)
+	typeItems := map[uint8]*[]*SummaryItem{
+		SummaryProject:  &s.Projects,
+		SummaryLanguage: &s.Languages,
+		SummaryEditor:   &s.Editors,
+		SummaryOS:       &s.OperatingSystems,
+		SummaryMachine:  &s.Machines,
+	}
+	var somePresentType uint8
+
+	for _, t := range types {
+		if len(*typeItems[t]) == 0 {
+			missingTypes = append(missingTypes, t)
+		} else {
+			somePresentType = t
+		}
+	}
+
+	// can't proceed if entire summary is empty
+	if len(missingTypes) == len(types) {
+		return
+	}
+
+	// calculate total duration from any of the present sets of items
+	var timeSum time.Duration
+	for _, item := range *typeItems[somePresentType] {
+		timeSum += item.Total
+	}
+
+	// construct dummy item for all missing types
+	for _, t := range missingTypes {
+		*typeItems[t] = append(*typeItems[t], &SummaryItem{
+			Type:  t,
+			Key:   UnknownSummaryKey,
+			Total: timeSum,
+		})
+	}
+}

routes/heartbeat.go

@@ -23,10 +23,15 @@ func NewHeartbeatHandler(heartbeatService *services.HeartbeatService) *Heartbeat
 	}
 }
 
+type heartbeatResponseVm struct {
+	Responses [][]interface{} `json:"responses"`
+}
+
 func (h *HeartbeatHandler) ApiPost(w http.ResponseWriter, r *http.Request) {
 	var heartbeats []*models.Heartbeat
 	user := r.Context().Value(models.UserKey).(*models.User)
 	opSys, editor, _ := utils.ParseUserAgent(r.Header.Get("User-Agent"))
+	machineName := r.Header.Get("X-Machine-Name")
 
 	dec := json.NewDecoder(r.Body)
 	if err := dec.Decode(&heartbeats); err != nil {
@@ -38,6 +43,7 @@ func (h *HeartbeatHandler) ApiPost(w http.ResponseWriter, r *http.Request) {
 	for _, hb := range heartbeats {
 		hb.OperatingSystem = opSys
 		hb.Editor = editor
+		hb.Machine = machineName
 		hb.User = user
 		hb.UserID = user.ID
 		hb.Augment(h.config.CustomLanguages)
@@ -55,5 +61,23 @@ func (h *HeartbeatHandler) ApiPost(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	w.WriteHeader(http.StatusOK)
+	utils.RespondJSON(w, http.StatusCreated, constructSuccessResponse(len(heartbeats)))
+}
+
+// construct weird response format (see https://github.com/wakatime/wakatime/blob/2e636d389bf5da4e998e05d5285a96ce2c181e3d/wakatime/api.py#L288)
+// to make the cli consider all heartbeats to having been successfully saved
+// response looks like: { "responses": [ [ { "data": {...} }, 201 ], ... ] }
+func constructSuccessResponse(n int) *heartbeatResponseVm {
+	responses := make([][]interface{}, n)
+
+	for i := 0; i < n; i++ {
+		r := make([]interface{}, 2)
+		r[0] = nil
+		r[1] = http.StatusCreated
+		responses[i] = r
+	}
+
+	return &heartbeatResponseVm{
+		Responses: responses,
+	}
 }

services/summary.go

@@ -65,12 +65,13 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco
 		heartbeats = append(heartbeats, hb...)
 	}
 
-	types := []uint8{models.SummaryProject, models.SummaryLanguage, models.SummaryEditor, models.SummaryOS}
+	types := []uint8{models.SummaryProject, models.SummaryLanguage, models.SummaryEditor, models.SummaryOS, models.SummaryMachine}
 
 	var projectItems []*models.SummaryItem
 	var languageItems []*models.SummaryItem
 	var editorItems []*models.SummaryItem
 	var osItems []*models.SummaryItem
+	var machineItems []*models.SummaryItem
 
 	if err := srv.AliasService.LoadUserAliases(user.ID); err != nil {
 		return nil, err
@@ -92,18 +93,40 @@ func (srv *SummaryService) Construct(from, to time.Time, user *models.User, reco
 			editorItems = item.Items
 		case models.SummaryOS:
 			osItems = item.Items
+		case models.SummaryMachine:
+			machineItems = item.Items
 		}
 	}
 	close(c)
 
+	realFrom, realTo := from, to
+	if len(existingSummaries) > 0 {
+		realFrom = existingSummaries[0].FromTime
+		realTo = existingSummaries[len(existingSummaries)-1].ToTime
+
+		for _, summary := range existingSummaries {
+			summary.FillUnknown()
+		}
+	}
+	if len(heartbeats) > 0 {
+		t1, t2 := time.Time(heartbeats[0].Time), time.Time(heartbeats[len(heartbeats)-1].Time)
+		if t1.After(realFrom) && t1.Before(time.Date(realFrom.Year(), realFrom.Month(), realFrom.Day()+1, 0, 0, 0, 0, realFrom.Location())) {
+			realFrom = t1
+		}
+		if t2.Before(realTo) && t2.After(time.Date(realTo.Year(), realTo.Month(), realTo.Day()-1, 0, 0, 0, 0, realTo.Location())) {
+			realTo = t2
+		}
+	}
+
 	aggregatedSummary := &models.Summary{
 		UserID:           user.ID,
-		FromTime:         from,
+		FromTime:         realFrom,
-		ToTime:           to,
+		ToTime:           realTo,
 		Projects:         projectItems,
 		Languages:        languageItems,
 		Editors:          editorItems,
 		OperatingSystems: osItems,
+		Machines:         machineItems,
 	}
 
 	allSummaries := []*models.Summary{aggregatedSummary}
@@ -134,10 +157,12 @@ func (srv *SummaryService) GetByUserWithin(user *models.User, from, to time.Time
 		Where(&models.Summary{UserID: user.ID}).
 		Where("from_time >= ?", from).
 		Where("to_time <= ?", to).
+		Order("from_time asc").
 		Preload("Projects", "type = ?", models.SummaryProject).
 		Preload("Languages", "type = ?", models.SummaryLanguage).
 		Preload("Editors", "type = ?", models.SummaryEditor).
 		Preload("OperatingSystems", "type = ?", models.SummaryOS).
+		Preload("Machines", "type = ?", models.SummaryMachine).
 		Find(&summaries).Error; err != nil {
 		return nil, err
 	}
@@ -171,10 +196,12 @@ func (srv *SummaryService) aggregateBy(heartbeats []*models.Heartbeat, summaryTy
 			key = h.Language
 		case models.SummaryOS:
 			key = h.OperatingSystem
+		case models.SummaryMachine:
+			key = h.Machine
 		}
 
 		if key == "" {
-			key = "unknown"
+			key = models.UnknownSummaryKey
 		}
 
 		if aliasedKey, err := srv.AliasService.GetAliasOrDefault(user.ID, summaryType, key); err == nil {
@@ -224,7 +251,16 @@ func getMissingIntervals(from, to time.Time, existingSummaries []*models.Summary
 
 	// Between
 	for i := 0; i < len(existingSummaries)-1; i++ {
-		if existingSummaries[i].ToTime.Before(existingSummaries[i+1].FromTime) {
+		t1, t2 := existingSummaries[i].ToTime, existingSummaries[i+1].FromTime
+		if t1.Equal(t2) {
+			continue
+		}
+
+		// round to end of day / start of day, assuming that summaries are always generated on a per-day basis
+		td1 := time.Date(t1.Year(), t1.Month(), t1.Day()+1, 0, 0, 0, 0, t1.Location())
+		td2 := time.Date(t2.Year(), t2.Month(), t2.Day(), 0, 0, 0, 0, t2.Location())
+		// one or more day missing in between?
+		if td1.Before(td2) {
 			intervals = append(intervals, &Interval{existingSummaries[i].ToTime, existingSummaries[i+1].FromTime})
 		}
 	}
@@ -251,6 +287,7 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
 		Languages:        make([]*models.SummaryItem, 0),
 		Editors:          make([]*models.SummaryItem, 0),
 		OperatingSystems: make([]*models.SummaryItem, 0),
+		Machines:         make([]*models.SummaryItem, 0),
 	}
 
 	for _, s := range summaries {
@@ -270,6 +307,7 @@ func mergeSummaries(summaries []*models.Summary) (*models.Summary, error) {
 		finalSummary.Languages = mergeSummaryItems(finalSummary.Languages, s.Languages)
 		finalSummary.Editors = mergeSummaryItems(finalSummary.Editors, s.Editors)
 		finalSummary.OperatingSystems = mergeSummaryItems(finalSummary.OperatingSystems, s.OperatingSystems)
+		finalSummary.Machines = mergeSummaryItems(finalSummary.Machines, s.Machines)
 	}
 
 	finalSummary.FromTime = minTime

static/assets/app.js

@@ -5,6 +5,7 @@ const projectsCanvas = document.getElementById('chart-projects')
 const osCanvas = document.getElementById('chart-os')
 const editorsCanvas = document.getElementById('chart-editor')
 const languagesCanvas = document.getElementById('chart-language')
+const machinesCanvas = document.getElementById('chart-machine')
 
 let charts = []
 let resizeCount = 0
@@ -56,7 +57,7 @@ function draw() {
                 .map(p => {
                     return {
                         label: p.key,
-                        data: [parseInt(p.total)],
+                        data: [parseInt(p.total) / 60],
                         backgroundColor: getRandomColor(p.key)
                     }
                 })
@@ -67,6 +68,14 @@ function draw() {
             legend: {
                 display: false
             },
+            scales: {
+                xAxes: [{
+                    scaleLabel: {
+                        display: true,
+                        labelString: 'Minutes'
+                    }
+                }]
+            },
             maintainAspectRatio: false,
             onResize: onChartResize
         }
@@ -135,10 +144,31 @@ function draw() {
         }
     })
 
+    let machineChart = new Chart(machinesCanvas.getContext('2d'), {
+        type: 'pie',
+        data: {
+            datasets: [{
+                data: wakapiData.machines
+                    .slice(0, Math.min(SHOW_TOP_N, wakapiData.machines.length))
+                    .map(p => parseInt(p.total)),
+                backgroundColor: wakapiData.machines.map(p => getRandomColor(p.key))
+            }],
+            labels: wakapiData.machines
+                .slice(0, Math.min(SHOW_TOP_N, wakapiData.machines.length))
+                .map(p => p.key)
+        },
+        options: {
+            title: Object.assign(titleOptions, {text: `Machines (top ${SHOW_TOP_N})`}),
+            tooltips: getTooltipOptions('machines', 'pie'),
+            maintainAspectRatio: false,
+            onResize: onChartResize
+        }
+    })
+
     getTotal(wakapiData.operatingSystems)
     document.getElementById('grid-container').style.visibility = 'visible'
 
-    charts = [projectChart, osChart, editorChart, languageChart]
+    charts = [projectChart, osChart, editorChart, languageChart, machineChart]
 
     charts.forEach(c => c.options.onResize(c.chart))
     equalizeHeights()

utils/date.go

@@ -15,12 +15,12 @@ func StartOfWeek() time.Time {
 
 func StartOfMonth() time.Time {
 	ref := time.Now()
-	return time.Date(ref.Year(), ref.Month(), 0, 0, 0, 0, 0, ref.Location())
+	return time.Date(ref.Year(), ref.Month(), 1, 0, 0, 0, 0, ref.Location())
 }
 
 func StartOfYear() time.Time {
 	ref := time.Now()
-	return time.Date(ref.Year(), time.January, 0, 0, 0, 0, 0, ref.Location())
+	return time.Date(ref.Year(), time.January, 1, 0, 0, 0, 0, ref.Location())
 }
 
 // https://stackoverflow.com/a/18632496

version.txt

@@ -1 +1 @@
-1.7.1
+1.8.1

views/foot.tpl.html

@@ -2,11 +2,13 @@
 <script src="https://cdnjs.cloudflare.com/ajax/libs/Chart.js/2.7.3/Chart.bundle.min.js"></script>
 
 <script>
+    const languageColors = {{ .LanguageColors | json }}
+
     let wakapiData = {}
-    let languageColors = {{ .LanguageColors | json }}
     wakapiData.projects = {{ .Projects | json }}
     wakapiData.operatingSystems = {{ .OperatingSystems | json }}
     wakapiData.editors = {{ .Editors | json }}
     wakapiData.languages = {{ .Languages | json }}
+    wakapiData.machines = {{ .Machines | json }}
 </script>
 <script src="assets/app.js"></script>

views/summary.tpl.html

@@ -78,6 +78,11 @@
                 <canvas id="chart-editor"></canvas>
             </div>
         </div>
+        <div class="w-full lg:w-1/2 p-1">
+            <div class="p-4 bg-white rounded shadow m-2" id="machine-container" style="height: 300px">
+                <canvas id="chart-machine"></canvas>
+            </div>
+        </div>
     </div>
 </main>