mirror
/
wakapi
mirror of https://github.com/muety/wakapi.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: implement file statistics (resolve #80)

This commit is contained in:
Ferdinand Mütsch 2023-03-22 20:45:27 +01:00
parent 0cf09a0871
commit 4ee3da6f7e
18 changed files with 1324 additions and 1138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2269
coverage/coverage.out
View File

File diff suppressed because it is too large Load Diff

3
helpers/summary.go
View File

@ -67,6 +67,9 @@ func ParseSummaryFilters(r *http.Request) *models.Filters {
if q := r.URL.Query().Get("branch"); q != "" {
filters.With(models.SummaryBranch, q)
}
if q := r.URL.Query().Get("entity"); q != "" {
filters.With(models.SummaryBranch, q)
}
return filters
}

2
models/compat/wakatime/v1/stats.go
View File

@ -77,6 +77,8 @@ func NewStatsFrom(summary *models.Summary, filters *models.Filters) *StatsViewMo
branches[i] = convertEntry(e, summary.TotalTimeBy(models.SummaryBranch))
}
// entities omitted intentionally
data.Editors = editors
data.Languages = languages
data.Machines = machines

19
models/compat/wakatime/v1/summaries.go
View File

@ -35,6 +35,7 @@ type SummariesData struct {
OperatingSystems []*SummariesEntry `json:"operating_systems"`
Projects []*SummariesEntry `json:"projects"`
Branches []*SummariesEntry `json:"branches,omitempty"`
Entities []*SummariesEntry `json:"entities,omitempty"`
GrandTotal *SummariesGrandTotal `json:"grand_total"`
Range *SummariesRange `json:"range"`
}
@ -132,8 +133,8 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
var wg sync.WaitGroup
wg.Add(6)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Projects {
@ -141,6 +142,7 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Editors {
@ -148,6 +150,7 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Languages {
@ -155,6 +158,7 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.OperatingSystems {
@ -162,6 +166,7 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Machines {
@ -169,6 +174,7 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Branches {
@ -176,9 +182,20 @@ func newDataFrom(s *models.Summary) *SummariesData {
}
}(data)
wg.Add(1)
go func(data *SummariesData) {
defer wg.Done()
for i, e := range s.Entities {
data.Entities[i] = convertEntry(e, s.TotalTimeBy(models.SummaryEntity))
}
}(data)
if s.Branches == nil {
data.Branches = nil
}
if s.Entities == nil {
data.Entities = nil
}
wg.Wait()
return data

25
models/duration.go
View File

@ -5,6 +5,7 @@ import (
"github.com/emvi/logbuch"
"github.com/mitchellh/hashstructure/v2"
"time"
"unicode"
)
type Duration struct {
@ -17,8 +18,24 @@ type Duration struct {
OperatingSystem string `json:"operating_system"`
Machine string `json:"machine"`
Branch string `json:"branch"`
Entity string `json:"entity"`
NumHeartbeats int `json:"-" hash:"ignore"`
GroupHash string `json:"-" hash:"ignore"`
excludeEntity bool `json:"-" hash:"ignore"`
}
func (d *Duration) HashInclude(field string, v interface{}) (bool, error) {
if field == "Entity" {
return !d.excludeEntity, nil
}
if field == "Time" ||
field == "Duration" ||
field == "NumHeartbeats" ||
field == "GroupHash" ||
unicode.IsLower(rune(field[0])) {
return false, nil
}
return true, nil
}
func NewDurationFromHeartbeat(h *Heartbeat) *Duration {
@ -32,11 +49,17 @@ func NewDurationFromHeartbeat(h *Heartbeat) *Duration {
OperatingSystem: h.OperatingSystem,
Machine: h.Machine,
Branch: h.Branch,
Entity: h.Entity,
NumHeartbeats: 1,
}
return d.Hashed()
}
func (d *Duration) WithEntityIgnored() *Duration {
d.excludeEntity = true
return d
}
func (d *Duration) Hashed() *Duration {
hash, err := hashstructure.Hash(d, hashstructure.FormatV2, nil)
if err != nil {
@ -60,6 +83,8 @@ func (d *Duration) GetKey(t uint8) (key string) {
key = d.Machine
case SummaryBranch:
key = d.Branch
case SummaryEntity:
key = d.Entity
}
if key == "" {

16
models/filters.go
View File

@ -14,6 +14,7 @@ type Filters struct {
Machine OrFilter
Label OrFilter
Branch OrFilter
Entity OrFilter
}
type OrFilter []string
@ -65,6 +66,8 @@ func (f *Filters) WithMultiple(entity uint8, keys []string) *Filters {
f.Label = append(f.Label, keys...)
case SummaryBranch:
f.Branch = append(f.Branch, keys...)
case SummaryEntity:
f.Entity = append(f.Entity, keys...)
}
return f
}
@ -84,6 +87,8 @@ func (f *Filters) One() (bool, uint8, OrFilter) {
return true, SummaryLabel, f.Label
} else if f.Branch != nil && f.Branch.Exists() {
return true, SummaryBranch, f.Branch
} else if f.Entity != nil && f.Entity.Exists() {
return true, SummaryEntity, f.Entity
}
return false, 0, OrFilter{}
}
@ -102,7 +107,7 @@ func (f *Filters) IsEmpty() bool {
func (f *Filters) Count() int {
var count int
for i := SummaryProject; i <= SummaryBranch; i++ {
for i := SummaryProject; i <= SummaryEntity; i++ {
count += f.CountByEntity(i)
}
return count
@ -114,7 +119,7 @@ func (f *Filters) CountByEntity(entity uint8) int {
func (f *Filters) EntityCount() int {
var count int
for i := SummaryProject; i <= SummaryBranch; i++ {
for i := SummaryProject; i <= SummaryEntity; i++ {
if c := f.CountByEntity(i); c > 0 {
count++
}
@ -138,6 +143,8 @@ func (f *Filters) ResolveEntity(entityId uint8) *OrFilter {
return &f.Label
case SummaryBranch:
return &f.Branch
case SummaryEntity:
return &f.Entity
default:
return &OrFilter{}
}
@ -209,6 +216,7 @@ func (f *Filters) WithAliases(resolve AliasReverseResolver) *Filters {
}
f.Branch = updated
}
// no aliases for entites / files
return f
}
@ -221,3 +229,7 @@ func (f *Filters) WithProjectLabels(resolve ProjectLabelReverseResolver) *Filter
}
return f
}
func (f *Filters) IsProjectDetails() bool {
return f != nil && f.Project != nil && f.Project.Exists()
}

2
models/heartbeat.go
View File

@ -63,6 +63,8 @@ func (h *Heartbeat) GetKey(t uint8) (key string) {
key = h.Machine
case SummaryBranch:
key = h.Branch
case SummaryEntity:
key = h.Entity
}
if key == "" {

11
models/summary.go
View File

@ -16,6 +16,7 @@ const (
SummaryMachine uint8 = 4
SummaryLabel uint8 = 5
SummaryBranch uint8 = 6
SummaryEntity uint8 = 7
)
const UnknownSummaryKey = "unknown"
@ -34,6 +35,7 @@ type Summary struct {
Machines SummaryItems `json:"machines" gorm:"constraint:OnUpdate:CASCADE,OnDelete:CASCADE"`
Labels SummaryItems `json:"labels" gorm:"-"` // labels are not persisted, but calculated at runtime, i.e. when summary is retrieved
Branches SummaryItems `json:"branches" gorm:"-"` // branches are not persisted, but calculated at runtime in case a project filter is applied
Entities SummaryItems `json:"entities" gorm:"-"` // entities are not persisted, but calculated at runtime in case a project filter is applied
NumHeartbeats int `json:"-"`
}
@ -62,11 +64,11 @@ type SummaryParams struct {
}
func SummaryTypes() []uint8 {
return []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine, SummaryLabel, SummaryBranch}
return []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine, SummaryLabel, SummaryBranch, SummaryEntity}
}
func NativeSummaryTypes() []uint8 {
return []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine, SummaryBranch}
return []uint8{SummaryProject, SummaryLanguage, SummaryEditor, SummaryOS, SummaryMachine, SummaryBranch, SummaryEntity}
}
func PersistedSummaryTypes() []uint8 {
@ -81,6 +83,7 @@ func (s *Summary) Sorted() *Summary {
sort.Sort(sort.Reverse(s.Editors))
sort.Sort(sort.Reverse(s.Labels))
sort.Sort(sort.Reverse(s.Branches))
sort.Sort(sort.Reverse(s.Entities))
return s
}
@ -97,6 +100,7 @@ func (s *Summary) MappedItems() map[uint8]*SummaryItems {
SummaryMachine: &s.Machines,
SummaryLabel: &s.Labels,
SummaryBranch: &s.Branches,
SummaryEntity: &s.Entities,
}
}
@ -116,6 +120,8 @@ func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
return &s.Labels
case SummaryBranch:
return &s.Branches
case SummaryEntity:
return &s.Entities
}
return nil
}
@ -325,6 +331,7 @@ func (s *Summary) WithResolvedAliases(resolve AliasResolver) *Summary {
s.Machines = processAliases(s.Machines)
s.Labels = processAliases(s.Labels)
s.Branches = processAliases(s.Branches)
// no aliases for entities / files
return s
}

3
routes/routes.go
View File

@ -90,6 +90,9 @@ func typeName(t uint8) string {
if t == models.SummaryBranch {
return "branch"
}
if t == models.SummaryEntity {
return "entity"
}
return "unknown"
}

3
services/duration.go
View File

@ -51,6 +51,9 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *
}
d1 := models.NewDurationFromHeartbeat(h)
if !filters.IsProjectDetails() {
d1 = d1.WithEntityIgnored() // only for efficiency
}
if list, ok := mapping[d1.GroupHash]; !ok || len(list) < 1 {
mapping[d1.GroupHash] = []*models.Duration{d1}

2
services/duration_test.go
View File

@ -26,6 +26,8 @@ const (
TestOsWin = "Windows"
TestMachine1 = "muety-desktop"
TestMachine2 = "muety-work"
TestEntity1 = "/home/bob/dev/wakapi.go"
TestEntity2 = "/home/bob/dev/SomethingElse.java"
TestBranchMaster = "master"
TestBranchDev = "dev"
MinUnixTime1 = 1601510400000 * 1e6

1
services/heartbeat.go
View File

@ -238,6 +238,7 @@ func (srv *HeartbeatService) updateEntityUserCacheByHeartbeat(hb *models.Heartbe
go srv.updateEntityUserCache(models.SummaryOS, hb.OperatingSystem, hb.User)
go srv.updateEntityUserCache(models.SummaryMachine, hb.Machine, hb.User)
go srv.updateEntityUserCache(models.SummaryBranch, hb.Branch, hb.User)
go srv.updateEntityUserCache(models.SummaryEntity, hb.Entity, hb.User)
}
func (srv *HeartbeatService) notifyBatch(heartbeats []*models.Heartbeat) {

12
services/summary.go
View File

@ -84,8 +84,9 @@ func (srv *SummaryService) Aliased(from, to time.Time, user *models.User, f type
summary.FillBy(models.SummaryProject, models.SummaryLabel) // first fill up labels from projects
summary.FillMissing() // then, full up types which are entirely missing
if withBranches := filters != nil && filters.Project != nil && filters.Project.Exists(); !withBranches {
if withDetails := filters != nil && filters.IsProjectDetails(); !withDetails {
summary.Branches = nil
summary.Entities = nil
}
srv.cache.SetDefault(cacheKey, summary)
@ -139,8 +140,9 @@ func (srv *SummaryService) Summarize(from, to time.Time, user *models.User, filt
}
types := models.PersistedSummaryTypes()
if filters != nil && filters.Project != nil && filters.Project.Exists() {
if filters != nil && filters.IsProjectDetails() {
types = append(types, models.SummaryBranch)
types = append(types, models.SummaryEntity)
}
typedAggregations := make(chan models.SummaryItemContainer)
@ -156,6 +158,7 @@ func (srv *SummaryService) Summarize(from, to time.Time, user *models.User, filt
var osItems []*models.SummaryItem
var machineItems []*models.SummaryItem
var branchItems []*models.SummaryItem
var entityItems []*models.SummaryItem
for i := 0; i < len(types); i++ {
item := <-typedAggregations
@ -172,6 +175,8 @@ func (srv *SummaryService) Summarize(from, to time.Time, user *models.User, filt
machineItems = item.Items
case models.SummaryBranch:
branchItems = item.Items
case models.SummaryEntity:
entityItems = item.Items
}
}
@ -190,6 +195,7 @@ func (srv *SummaryService) Summarize(from, to time.Time, user *models.User, filt
OperatingSystems: osItems,
Machines: machineItems,
Branches: branchItems,
Entities: entityItems,
NumHeartbeats: durations.TotalNumHeartbeats(),
}
@ -302,6 +308,7 @@ func (srv *SummaryService) mergeSummaries(summaries []*models.Summary) (*models.
Machines: make([]*models.SummaryItem, 0),
Labels: make([]*models.SummaryItem, 0),
Branches: make([]*models.SummaryItem, 0),
Entities: make([]*models.SummaryItem, 0),
}
var processed = map[time.Time]bool{}
@ -332,6 +339,7 @@ func (srv *SummaryService) mergeSummaries(summaries []*models.Summary) (*models.
finalSummary.Machines = srv.mergeSummaryItems(finalSummary.Machines, s.Machines)
finalSummary.Labels = srv.mergeSummaryItems(finalSummary.Labels, s.Labels)
finalSummary.Branches = srv.mergeSummaryItems(finalSummary.Branches, s.Branches)
finalSummary.Entities = srv.mergeSummaryItems(finalSummary.Entities, s.Entities)
finalSummary.NumHeartbeats += s.NumHeartbeats
processed[hash] = true

5
services/summary_test.go
View File

@ -43,6 +43,7 @@ func (suite *SummaryServiceTestSuite) SetupSuite() {
OperatingSystem: TestOsLinux,
Machine: TestMachine1,
Branch: TestBranchMaster,
Entity: TestEntity1,
Time: models.CustomTime(suite.TestStartTime),
Duration: 150 * time.Second,
NumHeartbeats: 2,
@ -55,6 +56,7 @@ func (suite *SummaryServiceTestSuite) SetupSuite() {
OperatingSystem: TestOsLinux,
Machine: TestMachine1,
Branch: TestBranchMaster,
Entity: TestEntity1,
Time: models.CustomTime(suite.TestStartTime.Add((30 + 130) * time.Second)),
Duration: 20 * time.Second,
NumHeartbeats: 1,
@ -67,6 +69,7 @@ func (suite *SummaryServiceTestSuite) SetupSuite() {
OperatingSystem: TestOsLinux,
Machine: TestMachine1,
Branch: TestBranchDev,
Entity: TestEntity1,
Time: models.CustomTime(suite.TestStartTime.Add(3 * time.Minute)),
Duration: 15 * time.Second,
NumHeartbeats: 3,
@ -154,6 +157,7 @@ func (suite *SummaryServiceTestSuite) TestSummaryService_Summarize() {
assert.Equal(suite.T(), 185*time.Second, result.TotalTimeBy(models.SummaryLanguage))
assert.Equal(suite.T(), 185*time.Second, result.TotalTimeBy(models.SummaryEditor))
assert.Zero(suite.T(), result.TotalTimeBy(models.SummaryBranch)) // no filters -> no branches contained
assert.Zero(suite.T(), result.TotalTimeBy(models.SummaryEntity)) // no filters -> no entities contained
assert.Zero(suite.T(), result.TotalTimeBy(models.SummaryLabel))
assert.Equal(suite.T(), 170*time.Second, result.TotalTimeByKey(models.SummaryEditor, TestEditorGoland))
assert.Equal(suite.T(), 15*time.Second, result.TotalTimeByKey(models.SummaryEditor, TestEditorVscode))
@ -477,6 +481,7 @@ func (suite *SummaryServiceTestSuite) TestSummaryService_Filters() {
result, _ := sut.Aliased(from, to, suite.TestUser, sut.Summarize, filters, false)
assert.NotNil(suite.T(), result.Branches) // project filters were applied -> include branches
assert.NotNil(suite.T(), result.Entities) // project filters were applied -> include entities
effectiveFilters := suite.DurationService.Calls[0].Arguments[3].(*models.Filters)
assert.Contains(suite.T(), effectiveFilters.Project, TestProject1) // because actually requested

2
static/assets/css/app.dist.css
View File

File diff suppressed because one or more lines are too long

BIN
static/assets/css/app.dist.css.br
View File

Binary file not shown.

70
static/assets/js/summary.js
View File

@ -13,6 +13,7 @@ const languagesCanvas = document.getElementById('chart-language')
const machinesCanvas = document.getElementById('chart-machine')
const labelsCanvas = document.getElementById('chart-label')
const branchesCanvas = document.getElementById('chart-branches')
const entitiesCanvas = document.getElementById('chart-entities')
const projectContainer = document.getElementById('project-container')
const osContainer = document.getElementById('os-container')
@ -21,10 +22,11 @@ const languageContainer = document.getElementById('language-container')
const machineContainer = document.getElementById('machine-container')
const labelContainer = document.getElementById('label-container')
const branchContainer = document.getElementById('branch-container')
const entityContainer = document.getElementById('entity-container')
const containers = [projectContainer, osContainer, editorContainer, languageContainer, machineContainer, labelContainer, branchContainer]
const canvases = [projectsCanvas, osCanvas, editorsCanvas, languagesCanvas, machinesCanvas, labelsCanvas, branchesCanvas]
const data = [wakapiData.projects, wakapiData.operatingSystems, wakapiData.editors, wakapiData.languages, wakapiData.machines, wakapiData.labels, wakapiData.branches]
const containers = [projectContainer, osContainer, editorContainer, languageContainer, machineContainer, labelContainer, branchContainer, entityContainer]
const canvases = [projectsCanvas, osCanvas, editorsCanvas, languagesCanvas, machinesCanvas, labelsCanvas, branchesCanvas, entitiesCanvas]
const data = [wakapiData.projects, wakapiData.operatingSystems, wakapiData.editors, wakapiData.languages, wakapiData.machines, wakapiData.labels, wakapiData.branches, wakapiData.entities]
let topNPickers = [...document.getElementsByClassName('top-picker')]
topNPickers.sort(((a, b) => parseInt(a.attributes['data-entity'].value) - parseInt(b.attributes['data-entity'].value)))
@ -378,6 +380,62 @@ function draw(subselection) {
})
: null
let entityChart = entitiesCanvas && !entitiesCanvas.classList.contains('hidden') && shouldUpdate(7)
? new Chart(entitiesCanvas.getContext('2d'), {
//type: 'horizontalBar',
type: "bar",
data: {
datasets: [{
data: wakapiData.entities
.slice(0, Math.min(showTopN[7], wakapiData.entities.length))
.map(p => parseInt(p.total)),
backgroundColor: wakapiData.entities.map((p, i) => {
const c = hexToRgb(vibrantColors ? getRandomColor(p.key) : getColor(p.key, i % baseColors.length))
return `rgba(${c.r}, ${c.g}, ${c.b}, 1)`
}),
hoverBackgroundColor: wakapiData.entities.map((p, i) => {
const c = hexToRgb(vibrantColors ? getRandomColor(p.key) : getColor(p.key, i % baseColors.length))
return `rgba(${c.r}, ${c.g}, ${c.b}, 0.8)`
}),
}],
labels: wakapiData.entities
.slice(0, Math.min(showTopN[7], wakapiData.entities.length))
.map(p => extractFile(p.key))
},
options: {
indexAxis: 'y',
scales: {
xAxes: {
title: {
display: true,
text: 'Duration (hh:mm:ss)',
},
ticks: {
callback: (label) => label.toString().toHHMMSS(),
}
}
},
plugins: {
legend: {
display: false,
},
tooltip: getTooltipOptions('entities'),
},
maintainAspectRatio: false,
onClick: (event, data) => {
const idx = data[0].index
const name = wakapiData.entities[idx].key
const url = new URL(window.location.href)
url.searchParams.set('project', name)
window.location.href = url.href
},
onHover: (event, elem) => {
event.native.target.style.cursor = elem[0] ? 'pointer' : 'default'
}
}
})
: null
charts[0] = projectChart ? projectChart : charts[0]
charts[1] = osChart ? osChart : charts[1]
charts[2] = editorChart ? editorChart : charts[2]
@ -385,6 +443,7 @@ function draw(subselection) {
charts[4] = machineChart ? machineChart : charts[4]
charts[5] = labelChart ? labelChart : charts[5]
charts[6] = branchChart ? branchChart : charts[6]
charts[7] = entityChart ? entityChart : charts[7]
}
function parseTopN() {
@ -447,6 +506,11 @@ function swapCharts(showEntity, hideEntity) {
document.getElementById(`${hideEntity}-container`).parentElement.classList.add('hidden')
}
function extractFile(filePath) {
const delimiter = filePath.includes('\\') ? '\\' : '/' // windows style path?
return filePath.split(delimiter).at(-1)
}
window.addEventListener('load', function () {
topNPickers.forEach(e => e.addEventListener('change', () => {
parseTopN()

17
views/summary.tpl.html
View File

@ -169,7 +169,7 @@
</div>
</div>
<div style="max-width: 100vw;">
<div style="max-width: 100vw;" class="{{ if .IsProjectDetails }} hidden {{ end }}">
<div class="p-4 px-6 pb-10 bg-gray-850 text-gray-300 rounded-md shadow flex flex-col {{ if .IsProjectDetails }} hidden {{ end }}" id="label-container" style="max-height: 300px">
<div class="flex justify-between text-lg" style="margin-bottom: -10px">
<span class="font-semibold whitespace-nowrap">Labels</span>
@ -186,6 +186,19 @@
</div>
</div>
</div>
<div class="p-4 px-6 pb-10 bg-gray-850 text-gray-300 rounded-md shadow flex flex-col {{ if not .IsProjectDetails }} hidden {{ end }} col-span-2" id="entity-container" style="max-height: 500px">
<div class="flex justify-between">
<span class="font-semibold text-lg w-1/2 flex-1 whitespace-nowrap">Files</span>
<div class="flex justify-end flex-1 text-xs items-center">
<input type="number" min="1" id="entity-top-picker" data-entity="2" class="top-picker bg-gray-800 rounded-md text-center w-12" value="10">
</div>
</div>
<canvas id="chart-entities" class="mt-4"></canvas>
<div class="hidden placeholder-container flex items-center justify-center h-full flex-col">
<span class="text-md font-semibold text-gray-500 mt-4">No data</span>
</div>
</div>
</div>
{{ else }}
@ -238,8 +251,10 @@
wakapiData.labels = {{ .Labels | json }}
{{ if .IsProjectDetails }}
wakapiData.branches = {{ .Branches | json }}
wakapiData.entities = {{ .Entities | json }}
{{ else }}
wakapiData.branches = []
wakapiData.entities = []
{{ end }}
</script>
<script src="assets/js/summary.js"></script>