chore: apply filters in database query (see #335)

mocks/heartbeat_service.go

@@ -40,6 +40,11 @@ func (m *HeartbeatServiceMock) GetAllWithin(time time.Time, time2 time.Time, use
 	return args.Get(0).([]*models.Heartbeat), args.Error(1)
 }
 
+func (m *HeartbeatServiceMock) GetAllWithinByFilters(time time.Time, time2 time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	args := m.Called(time, time2, user, filters)
+	return args.Get(0).([]*models.Heartbeat), args.Error(1)
+}
+
 func (m *HeartbeatServiceMock) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	args := m.Called()
 	return args.Get(0).([]*models.TimeByUser), args.Error(1)

models/filters.go

@@ -92,7 +92,7 @@ func (f *Filters) OneOrEmpty() FilterElement {
 	if ok, t, of := f.One(); ok {
 		return FilterElement{entity: t, filter: of}
 	}
-	return FilterElement{}
+	return FilterElement{entity: SummaryUnknown, filter: []string{}}
 }
 
 func (f *Filters) IsEmpty() bool {
@@ -100,6 +100,49 @@ func (f *Filters) IsEmpty() bool {
 	return !nonEmpty
 }
 
+func (f *Filters) Count() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		count += f.CountByEntity(i)
+	}
+	return count
+}
+
+func (f *Filters) CountByEntity(entity uint8) int {
+	return len(*f.ResolveEntity(entity))
+}
+
+func (f *Filters) EntityCount() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		if c := f.CountByEntity(i); c > 0 {
+			count++
+		}
+	}
+	return count
+}
+
+func (f *Filters) ResolveEntity(entityId uint8) *OrFilter {
+	switch entityId {
+	case SummaryProject:
+		return &f.Project
+	case SummaryLanguage:
+		return &f.Language
+	case SummaryEditor:
+		return &f.Editor
+	case SummaryOS:
+		return &f.OS
+	case SummaryMachine:
+		return &f.Machine
+	case SummaryLabel:
+		return &f.Label
+	case SummaryBranch:
+		return &f.Branch
+	default:
+		return &OrFilter{}
+	}
+}
+
 func (f *Filters) Hash() string {
 	hash, err := hashstructure.Hash(f, hashstructure.FormatV2, nil)
 	if err != nil {

models/heartbeat.go

@@ -15,18 +15,18 @@ type Heartbeat struct {
 	Entity          string     `json:"entity" gorm:"not null; index:idx_entity"`
 	Type            string     `json:"type"`
 	Category        string     `json:"category"`
-	Project         string     `json:"project"`
+	Project         string     `json:"project" gorm:"index:idx_project"`
-	Branch          string     `json:"branch"`
+	Branch          string     `json:"branch" gorm:"index:idx_branch"`
 	Language        string     `json:"language" gorm:"index:idx_language"`
 	IsWrite         bool       `json:"is_write"`
-	Editor          string     `json:"editor" hash:"ignore"`           // ignored because editor might be parsed differently by wakatime
+	Editor          string     `json:"editor" gorm:"index:idx_editor" hash:"ignore"`                     // ignored because editor might be parsed differently by wakatime
-	OperatingSystem string     `json:"operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
+	OperatingSystem string     `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
-	Machine         string     `json:"machine" hash:"ignore"`          // ignored because wakatime api doesn't return machines currently
+	Machine         string     `json:"machine" gorm:"index:idx_machine" hash:"ignore"`                   // ignored because wakatime api doesn't return machines currently
-	UserAgent       string     `json:"user_agent" hash:"ignore"`
+	UserAgent       string     `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
 	Time            CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"`
 	Hash            string     `json:"-" gorm:"type:varchar(17); uniqueIndex"`
-	Origin          string     `json:"-" hash:"ignore"`
+	Origin          string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
-	OriginId        string     `json:"-" hash:"ignore"`
+	OriginId        string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
 	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
 }
 
@@ -99,3 +99,15 @@ func (h *Heartbeat) Hashed() *Heartbeat {
 	h.Hash = fmt.Sprintf("%x", hash) // "uint64 values with high bit set are not supported"
 	return h
 }
+
+func GetEntityColumn(t uint8) string {
+	return []string{
+		"project",
+		"language",
+		"editor",
+		"operating_system",
+		"machine",
+		"label",
+		"branch",
+	}[t]
+}

models/summary.go

@@ -8,6 +8,7 @@ import (
 
 const (
 	NSummaryTypes   uint8 = 99
+	SummaryUnknown  uint8 = 98
 	SummaryProject  uint8 = 0
 	SummaryLanguage uint8 = 1
 	SummaryEditor   uint8 = 2
@@ -103,6 +104,20 @@ func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
 	return s.MappedItems()[summaryType]
 }
 
+func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {
+	if len(types) == 0 {
+		return s
+	}
+
+	for _, t := range SummaryTypes() {
+		if keep, ok := types[t]; !keep || !ok {
+			*s.ItemsByType(t) = []*SummaryItem{}
+		}
+	}
+
+	return s
+}
+
 /* Augments the summary in a way that at least one item is present for every type.
 If a summary has zero items for a given type, but one or more for any of the other types,
 the total summary duration can be derived from those and inserted as a dummy-item with key "unknown"

models/summary_test.go

@@ -168,6 +168,66 @@ func TestSummary_WithResolvedAliases(t *testing.T) {
 	assert.Empty(t, sut.Machines)
 }
 
+func TestSummary_KeepOnly(t *testing.T) {
+	newSummary := func() *Summary {
+		return &Summary{
+			Projects: []*SummaryItem{
+				{
+					Type: SummaryProject,
+					Key:  "wakapi",
+					// hack to work around the issue that the total time of a summary item is mistakenly represented in seconds
+					Total: 10 * time.Minute / time.Second,
+				},
+				{
+					Type:  SummaryProject,
+					Key:   "anchr",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Languages: []*SummaryItem{
+				{
+					Type:  SummaryLanguage,
+					Key:   "Go",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Editors: []*SummaryItem{
+				{
+					Type:  SummaryEditor,
+					Key:   "VSCode",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+		}
+	}
+
+	var sut *Summary
+
+	sut = newSummary().KeepOnly(map[uint8]bool{}) // keep all
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.Zero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.Zero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryEditor: true, SummaryLanguage: true})
+	assert.Zero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 10*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	sut.FillMissing()
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+}
+
 func TestSummaryItems_Sorted(t *testing.T) {
 	testDuration1, testDuration2, testDuration3 := 10*time.Minute, 5*time.Minute, 20*time.Minute
 

repositories/heartbeat.go

@@ -77,6 +77,26 @@ func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User
 	return heartbeats, nil
 }
 
+func (r *HeartbeatRepository) GetAllWithinByFilters(from, to time.Time, user *models.User, filterMap map[string][]string) ([]*models.Heartbeat, error) {
+	// https://stackoverflow.com/a/20765152/3112139
+	var heartbeats []*models.Heartbeat
+
+	q := r.db.
+		Where(&models.Heartbeat{UserID: user.ID}).
+		Where("time >= ?", from.Local()).
+		Where("time < ?", to.Local()).
+		Order("time asc")
+
+	for col, vals := range filterMap {
+		q = q.Where(col+" in ?", vals)
+	}
+
+	if err := q.Find(&heartbeats).Error; err != nil {
+		return nil, err
+	}
+	return heartbeats, nil
+}
+
 func (r *HeartbeatRepository) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	var result []*models.TimeByUser
 	r.db.Model(&models.User{}).

repositories/repositories.go

@@ -20,6 +20,7 @@ type IHeartbeatRepository interface {
 	InsertBatch([]*models.Heartbeat) error
 	GetAll() ([]*models.Heartbeat, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, map[string][]string) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLastByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)

services/duration.go

@@ -22,7 +22,15 @@ func NewDurationService(heartbeatService IHeartbeatService) *DurationService {
 }
 
 func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *models.Filters) (models.Durations, error) {
-	heartbeats, err := srv.heartbeatService.GetAllWithin(from, to, user)
+	get := srv.heartbeatService.GetAllWithin
+
+	if filters != nil && !filters.IsEmpty() {
+		get = func(t1 time.Time, t2 time.Time, user *models.User) ([]*models.Heartbeat, error) {
+			return srv.heartbeatService.GetAllWithinByFilters(t1, t2, user, filters)
+		}
+	}
+
+	heartbeats, err := get(from, to, user)
 	if err != nil {
 		return nil, err
 	}

services/duration_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/muety/wakapi/mocks"
 	"github.com/muety/wakapi/models"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 	"math/rand"
 	"testing"
@@ -175,7 +176,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get_Filtered() {
 	)
 
 	from, to = suite.TestStartTime.Add(-1*time.Hour), suite.TestStartTime.Add(1*time.Hour)
-	suite.HeartbeatService.On("GetAllWithin", from, to, suite.TestUser).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
+	suite.HeartbeatService.On("GetAllWithinByFilters", from, to, suite.TestUser, mock.Anything).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
 
 	durations, err = sut.Get(from, to, suite.TestUser, models.NewFiltersWith(models.SummaryEditor, TestEditorGoland))
 	assert.Nil(suite.T(), err)

services/heartbeat.go

@@ -134,6 +134,14 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
 	return srv.augmented(heartbeats, user.ID)
 }
 
+func (srv *HeartbeatService) GetAllWithinByFilters(from, to time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	heartbeats, err := srv.repository.GetAllWithinByFilters(from, to, user, srv.filtersToColumnMap(filters))
+	if err != nil {
+		return nil, err
+	}
+	return srv.augmented(heartbeats, user.ID)
+}
+
 func (srv *HeartbeatService) GetLatestByUser(user *models.User) (*models.Heartbeat, error) {
 	return srv.repository.GetLatestByUser(user)
 }
@@ -237,3 +245,14 @@ func (srv *HeartbeatService) countTotalCacheKey() string {
 func (srv *HeartbeatService) countCacheTtl() time.Duration {
 	return time.Duration(srv.config.App.CountCacheTTLMin) * time.Minute
 }
+
+func (srv *HeartbeatService) filtersToColumnMap(filters *models.Filters) map[string][]string {
+	columnMap := map[string][]string{}
+	for _, t := range models.SummaryTypes() {
+		f := filters.ResolveEntity(t)
+		if len(*f) > 0 {
+			columnMap[models.GetEntityColumn(t)] = *f
+		}
+	}
+	return columnMap
+}

services/services.go

@@ -33,6 +33,7 @@ type IHeartbeatService interface {
 	CountByUser(*models.User) (int64, error)
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, *models.Filters) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)