chore: remove entity index again

chore: add migration note

Dockerfile

@@ -56,4 +56,6 @@ COPY --from=build-env /app .
 
 VOLUME /data
 
+EXPOSE 3000
+
 ENTRYPOINT /app/entrypoint.sh

README.md

@@ -154,10 +154,16 @@ You can specify configuration options either via a config file (default: `config
 | `db.ssl` /<br> `WAKAPI_DB_SSL`                                               | `false`                                          | Whether to use TLS encryption for database connection (Postgres and CockroachDB only)                                                                                    |
 | `db.automgirate_fail_silently` /<br> `WAKAPI_DB_AUTOMIGRATE_FAIL_SILENTLY`   | `false`                                          | Whether to ignore schema auto-migration failures when starting up                                                                                                        |
 | `mail.enabled` /<br> `WAKAPI_MAIL_ENABLED`                                   | `true`                                           | Whether to allow Wakapi to send e-mail (e.g. for password resets)                                                                                                        |
-| `mail.sender` /<br> `WAKAPI_MAIL_SENDER`                                     | `noreply@wakapi.dev`                             | Default sender address for outgoing mails (ignored for MailWhale)                                                                                                        |
+| `mail.sender` /<br> `WAKAPI_MAIL_SENDER`                                     | `Wakapi <noreply@wakapi.dev>`                    | Default sender address for outgoing mails (ignored for MailWhale)                                                                                                        |
 | `mail.provider` /<br> `WAKAPI_MAIL_PROVIDER`                                 | `smtp`                                           | Implementation to use for sending mails (one of [`smtp`, `mailwhale`])                                                                                                   |
-| `mail.smtp.*` /<br> `WAKAPI_MAIL_SMTP_*`                                     | `-`                                              | Various options to configure SMTP. See [default config](config.default.yml) for details                                                                                  |
-| `mail.mailwhale.*` /<br> `WAKAPI_MAIL_MAILWHALE_*`                           | `-`                                              | Various options to configure [MailWhale](https://mailwhale.dev) sending service. See [default config](config.default.yml) for details                                    |
+| `mail.smtp.host` /<br> `WAKAPI_MAIL_SMTP_HOST`                               | -                                                | SMTP server address for sending mail (if using `smtp` mail provider)                                                                                                     |
+| `mail.smtp.port` /<br> `WAKAPI_MAIL_SMTP_PORT`                               | -                                                | SMTP server port (usually 465)                                                                                                                                           |
+| `mail.smtp.username` /<br> `WAKAPI_MAIL_SMTP_USER`                           | -                                                | SMTP server authentication username                                                                                                                                      |
+| `mail.smtp.password` /<br> `WAKAPI_MAIL_SMTP_PASS`                           | -                                                | SMTP server authentication password                                                                                                                                      |
+| `mail.smtp.tls` /<br> `WAKAPI_MAIL_SMTP_TLS`                                 | `false`                                          | Whether the SMTP server requires TLS encryption (`false` for STARTTLS or no encryption)                                                                                  |
+| `mail.mailwhale.url` /<br> `WAKAPI_MAIL_MAILWHALE_URL`                       | -                                                | URL of [MailWhale](https://mailwhale.dev) instance (e.g. `https://mailwhale.dev`) (if using `mailwhale` mail provider`)                                                  |
+| `mail.mailwhale.client_id` /<br> `WAKAPI_MAIL_MAILWHALE_CLIENT_ID`           | -                                                | MailWhale API client ID                                                                                                                                                  |
+| `mail.mailwhale.client_secret` /<br> `WAKAPI_MAIL_MAILWHALE_CLIENT_SECRET`   | -                                                | MailWhale API client secret                                                                                                                                              |
 | `sentry.dsn` /<br> `WAKAPI_SENTRY_DSN`                                       | –                                                | DSN for to integrate [Sentry](https://sentry.io) for error logging and tracing (leave empty to disable)                                                                  |
 | `sentry.enable_tracing` /<br> `WAKAPI_SENTRY_TRACING`                        | `false`                                          | Whether to enable Sentry request tracing                                                                                                                                 |
 | `sentry.sample_rate` /<br> `WAKAPI_SENTRY_SAMPLE_RATE`                       | `0.75`                                           | Probability of tracing a request in Sentry                                                                                                                               |

migrations/20220313_index_generation_hint.go

@@ -0,0 +1,24 @@
+package migrations
+
+import (
+	"github.com/emvi/logbuch"
+	"github.com/muety/wakapi/config"
+	"gorm.io/gorm"
+)
+
+func init() {
+	const name = "20220313-index_generation_hint"
+	f := migrationFunc{
+		name: name,
+		f: func(db *gorm.DB, cfg *config.Config) error {
+			if hasRun(name, db) {
+				return nil
+			}
+			logbuch.Info("please note: the following migrations might take a few minutes, as column types are changed and new indexes are created, have some patience")
+			setHasRun(name, db)
+			return nil
+		},
+	}
+
+	registerPreMigration(f)
+}

mocks/heartbeat_service.go

@@ -40,6 +40,11 @@ func (m *HeartbeatServiceMock) GetAllWithin(time time.Time, time2 time.Time, use
 	return args.Get(0).([]*models.Heartbeat), args.Error(1)
 }
 
+func (m *HeartbeatServiceMock) GetAllWithinByFilters(time time.Time, time2 time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	args := m.Called(time, time2, user, filters)
+	return args.Get(0).([]*models.Heartbeat), args.Error(1)
+}
+
 func (m *HeartbeatServiceMock) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	args := m.Called()
 	return args.Get(0).([]*models.TimeByUser), args.Error(1)

models/filters.go

@@ -92,7 +92,7 @@ func (f *Filters) OneOrEmpty() FilterElement {
 	if ok, t, of := f.One(); ok {
 		return FilterElement{entity: t, filter: of}
 	}
-	return FilterElement{}
+	return FilterElement{entity: SummaryUnknown, filter: []string{}}
 }
 
 func (f *Filters) IsEmpty() bool {
@@ -100,6 +100,49 @@ func (f *Filters) IsEmpty() bool {
 	return !nonEmpty
 }
 
+func (f *Filters) Count() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		count += f.CountByEntity(i)
+	}
+	return count
+}
+
+func (f *Filters) CountByEntity(entity uint8) int {
+	return len(*f.ResolveEntity(entity))
+}
+
+func (f *Filters) EntityCount() int {
+	var count int
+	for i := SummaryProject; i <= SummaryBranch; i++ {
+		if c := f.CountByEntity(i); c > 0 {
+			count++
+		}
+	}
+	return count
+}
+
+func (f *Filters) ResolveEntity(entityId uint8) *OrFilter {
+	switch entityId {
+	case SummaryProject:
+		return &f.Project
+	case SummaryLanguage:
+		return &f.Language
+	case SummaryEditor:
+		return &f.Editor
+	case SummaryOS:
+		return &f.OS
+	case SummaryMachine:
+		return &f.Machine
+	case SummaryLabel:
+		return &f.Label
+	case SummaryBranch:
+		return &f.Branch
+	default:
+		return &OrFilter{}
+	}
+}
+
 func (f *Filters) Hash() string {
 	hash, err := hashstructure.Hash(f, hashstructure.FormatV2, nil)
 	if err != nil {

models/heartbeat.go

@@ -12,21 +12,21 @@ type Heartbeat struct {
 	ID              uint64     `gorm:"primary_key" hash:"ignore"`
 	User            *User      `json:"-" gorm:"not null; constraint:OnUpdate:CASCADE,OnDelete:CASCADE;" hash:"ignore"`
 	UserID          string     `json:"-" gorm:"not null; index:idx_time_user"`
-	Entity          string     `json:"entity" gorm:"not null; index:idx_entity"`
+	Entity          string     `json:"entity" gorm:"not null"`
 	Type            string     `json:"type"`
 	Category        string     `json:"category"`
-	Project         string     `json:"project"`
-	Branch          string     `json:"branch"`
+	Project         string     `json:"project" gorm:"index:idx_project"`
+	Branch          string     `json:"branch" gorm:"index:idx_branch"`
 	Language        string     `json:"language" gorm:"index:idx_language"`
 	IsWrite         bool       `json:"is_write"`
-	Editor          string     `json:"editor" hash:"ignore"`           // ignored because editor might be parsed differently by wakatime
-	OperatingSystem string     `json:"operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
-	Machine         string     `json:"machine" hash:"ignore"`          // ignored because wakatime api doesn't return machines currently
-	UserAgent       string     `json:"user_agent" hash:"ignore"`
+	Editor          string     `json:"editor" gorm:"index:idx_editor" hash:"ignore"`                     // ignored because editor might be parsed differently by wakatime
+	OperatingSystem string     `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime
+	Machine         string     `json:"machine" gorm:"index:idx_machine" hash:"ignore"`                   // ignored because wakatime api doesn't return machines currently
+	UserAgent       string     `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"`
 	Time            CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"`
 	Hash            string     `json:"-" gorm:"type:varchar(17); uniqueIndex"`
-	Origin          string     `json:"-" hash:"ignore"`
-	OriginId        string     `json:"-" hash:"ignore"`
+	Origin          string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
+	OriginId        string     `json:"-" hash:"ignore" gorm:"type:varchar(255)"`
 	CreatedAt       CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt
 }
 
@@ -99,3 +99,15 @@ func (h *Heartbeat) Hashed() *Heartbeat {
 	h.Hash = fmt.Sprintf("%x", hash) // "uint64 values with high bit set are not supported"
 	return h
 }
+
+func GetEntityColumn(t uint8) string {
+	return []string{
+		"project",
+		"language",
+		"editor",
+		"operating_system",
+		"machine",
+		"label",
+		"branch",
+	}[t]
+}

models/summary.go

@@ -8,6 +8,7 @@ import (
 
 const (
 	NSummaryTypes   uint8 = 99
+	SummaryUnknown  uint8 = 98
 	SummaryProject  uint8 = 0
 	SummaryLanguage uint8 = 1
 	SummaryEditor   uint8 = 2
@@ -103,6 +104,20 @@ func (s *Summary) ItemsByType(summaryType uint8) *SummaryItems {
 	return s.MappedItems()[summaryType]
 }
 
+func (s *Summary) KeepOnly(types map[uint8]bool) *Summary {
+	if len(types) == 0 {
+		return s
+	}
+
+	for _, t := range SummaryTypes() {
+		if keep, ok := types[t]; !keep || !ok {
+			*s.ItemsByType(t) = []*SummaryItem{}
+		}
+	}
+
+	return s
+}
+
 /* Augments the summary in a way that at least one item is present for every type.
 If a summary has zero items for a given type, but one or more for any of the other types,
 the total summary duration can be derived from those and inserted as a dummy-item with key "unknown"

models/summary_test.go

@@ -168,6 +168,66 @@ func TestSummary_WithResolvedAliases(t *testing.T) {
 	assert.Empty(t, sut.Machines)
 }
 
+func TestSummary_KeepOnly(t *testing.T) {
+	newSummary := func() *Summary {
+		return &Summary{
+			Projects: []*SummaryItem{
+				{
+					Type: SummaryProject,
+					Key:  "wakapi",
+					// hack to work around the issue that the total time of a summary item is mistakenly represented in seconds
+					Total: 10 * time.Minute / time.Second,
+				},
+				{
+					Type:  SummaryProject,
+					Key:   "anchr",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Languages: []*SummaryItem{
+				{
+					Type:  SummaryLanguage,
+					Key:   "Go",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+			Editors: []*SummaryItem{
+				{
+					Type:  SummaryEditor,
+					Key:   "VSCode",
+					Total: 10 * time.Minute / time.Second,
+				},
+			},
+		}
+	}
+
+	var sut *Summary
+
+	sut = newSummary().KeepOnly(map[uint8]bool{}) // keep all
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.Zero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.Zero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 20*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryEditor: true, SummaryLanguage: true})
+	assert.Zero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+	assert.Equal(t, 10*time.Minute, sut.TotalTime())
+
+	sut = newSummary().KeepOnly(map[uint8]bool{SummaryProject: true})
+	sut.FillMissing()
+	assert.NotZero(t, sut.TotalTimeBy(SummaryProject))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryLanguage))
+	assert.NotZero(t, sut.TotalTimeBy(SummaryEditor))
+}
+
 func TestSummaryItems_Sorted(t *testing.T) {
 	testDuration1, testDuration2, testDuration3 := 10*time.Minute, 5*time.Minute, 20*time.Minute
 

repositories/heartbeat.go

@@ -1,7 +1,6 @@
 package repositories
 
 import (
-	"errors"
 	"github.com/muety/wakapi/models"
 	"gorm.io/gorm"
 	"gorm.io/gorm/clause"
@@ -77,6 +76,26 @@ func (r *HeartbeatRepository) GetAllWithin(from, to time.Time, user *models.User
 	return heartbeats, nil
 }
 
+func (r *HeartbeatRepository) GetAllWithinByFilters(from, to time.Time, user *models.User, filterMap map[string][]string) ([]*models.Heartbeat, error) {
+	// https://stackoverflow.com/a/20765152/3112139
+	var heartbeats []*models.Heartbeat
+
+	q := r.db.
+		Where(&models.Heartbeat{UserID: user.ID}).
+		Where("time >= ?", from.Local()).
+		Where("time < ?", to.Local()).
+		Order("time asc")
+
+	for col, vals := range filterMap {
+		q = q.Where(col+" in ?", vals)
+	}
+
+	if err := q.Find(&heartbeats).Error; err != nil {
+		return nil, err
+	}
+	return heartbeats, nil
+}
+
 func (r *HeartbeatRepository) GetFirstByUsers() ([]*models.TimeByUser, error) {
 	var result []*models.TimeByUser
 	r.db.Model(&models.User{}).
@@ -138,16 +157,10 @@ func (r *HeartbeatRepository) CountByUsers(users []*models.User) ([]*models.Coun
 }
 
 func (r HeartbeatRepository) GetEntitySetByUser(entityType uint8, user *models.User) ([]string, error) {
-	columns := []string{"project", "language", "editor", "operating_system", "machine"}
-	if int(entityType) >= len(columns) {
-		// invalid entity type
-		return nil, errors.New("invalid entity type")
-	}
-
 	var results []string
 	if err := r.db.
 		Model(&models.Heartbeat{}).
-		Distinct(columns[entityType]).
+		Distinct(models.GetEntityColumn(entityType)).
 		Where(&models.Heartbeat{UserID: user.ID}).
 		Find(&results).Error; err != nil {
 		return nil, err

repositories/repositories.go

@@ -20,6 +20,7 @@ type IHeartbeatRepository interface {
 	InsertBatch([]*models.Heartbeat) error
 	GetAll() ([]*models.Heartbeat, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, map[string][]string) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLastByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)

services/duration.go

@@ -22,7 +22,15 @@ func NewDurationService(heartbeatService IHeartbeatService) *DurationService {
 }
 
 func (srv *DurationService) Get(from, to time.Time, user *models.User, filters *models.Filters) (models.Durations, error) {
-	heartbeats, err := srv.heartbeatService.GetAllWithin(from, to, user)
+	get := srv.heartbeatService.GetAllWithin
+
+	if filters != nil && !filters.IsEmpty() {
+		get = func(t1 time.Time, t2 time.Time, user *models.User) ([]*models.Heartbeat, error) {
+			return srv.heartbeatService.GetAllWithinByFilters(t1, t2, user, filters)
+		}
+	}
+
+	heartbeats, err := get(from, to, user)
 	if err != nil {
 		return nil, err
 	}

services/duration_test.go

@@ -4,6 +4,7 @@ import (
 	"github.com/muety/wakapi/mocks"
 	"github.com/muety/wakapi/models"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 	"math/rand"
 	"testing"
@@ -175,7 +176,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get_Filtered() {
 	)
 
 	from, to = suite.TestStartTime.Add(-1*time.Hour), suite.TestStartTime.Add(1*time.Hour)
-	suite.HeartbeatService.On("GetAllWithin", from, to, suite.TestUser).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
+	suite.HeartbeatService.On("GetAllWithinByFilters", from, to, suite.TestUser, mock.Anything).Return(filterHeartbeats(from, to, suite.TestHeartbeats), nil)
 
 	durations, err = sut.Get(from, to, suite.TestUser, models.NewFiltersWith(models.SummaryEditor, TestEditorGoland))
 	assert.Nil(suite.T(), err)

services/heartbeat.go

@@ -134,6 +134,14 @@ func (srv *HeartbeatService) GetAllWithin(from, to time.Time, user *models.User)
 	return srv.augmented(heartbeats, user.ID)
 }
 
+func (srv *HeartbeatService) GetAllWithinByFilters(from, to time.Time, user *models.User, filters *models.Filters) ([]*models.Heartbeat, error) {
+	heartbeats, err := srv.repository.GetAllWithinByFilters(from, to, user, srv.filtersToColumnMap(filters))
+	if err != nil {
+		return nil, err
+	}
+	return srv.augmented(heartbeats, user.ID)
+}
+
 func (srv *HeartbeatService) GetLatestByUser(user *models.User) (*models.Heartbeat, error) {
 	return srv.repository.GetLatestByUser(user)
 }
@@ -237,3 +245,14 @@ func (srv *HeartbeatService) countTotalCacheKey() string {
 func (srv *HeartbeatService) countCacheTtl() time.Duration {
 	return time.Duration(srv.config.App.CountCacheTTLMin) * time.Minute
 }
+
+func (srv *HeartbeatService) filtersToColumnMap(filters *models.Filters) map[string][]string {
+	columnMap := map[string][]string{}
+	for _, t := range models.SummaryTypes() {
+		f := filters.ResolveEntity(t)
+		if len(*f) > 0 {
+			columnMap[models.GetEntityColumn(t)] = *f
+		}
+	}
+	return columnMap
+}

services/mail/smtp.go

@@ -49,6 +49,11 @@ func (s *SMTPSendingService) Send(mail *models.Mail) error {
 		if ok, _ := c.Extension("AUTH"); !ok {
 			return errors.New("smtp: server doesn't support AUTH")
 		}
+
+		if len(s.config.Username) == 0 || len(s.config.Password) == 0 {
+			return errors.New("smtp: server requires authentication, but no authentication is provided")
+		}
+
 		if err = c.Auth(s.auth); err != nil {
 			return err
 		}

services/services.go

@@ -33,6 +33,7 @@ type IHeartbeatService interface {
 	CountByUser(*models.User) (int64, error)
 	CountByUsers([]*models.User) ([]*models.CountByUser, error)
 	GetAllWithin(time.Time, time.Time, *models.User) ([]*models.Heartbeat, error)
+	GetAllWithinByFilters(time.Time, time.Time, *models.User, *models.Filters) ([]*models.Heartbeat, error)
 	GetFirstByUsers() ([]*models.TimeByUser, error)
 	GetLatestByUser(*models.User) (*models.Heartbeat, error)
 	GetLatestByOriginAndUser(string, *models.User) (*models.Heartbeat, error)

static/assets/js/summary.js

@@ -132,9 +132,9 @@ function draw(subselection) {
                 onClick: (event, data) => {
                     const idx = data[0].index
                     const name = wakapiData.projects[idx].key
-                    const query = new URLSearchParams(window.location.search)
-                    query.set('project', name)
-                    window.location.replace(`${window.location.pathname.slice(1)}?${query.toString()}`)
+                    const url = new URL(window.location.href)
+                    url.searchParams.set('project', name)
+                    window.location.href = url.href
                 },
                 onHover: (event, elem) => {
                     event.native.target.style.cursor = elem[0] ? 'pointer' : 'default'

version.txt

@@ -1 +1 @@
-2.2.2
+2.2.5

views/summary.tpl.html

@@ -189,12 +189,11 @@
                 # <strong>Step 1:</strong> Download WakaTime plugin for your IDE<br>
                 # See: https://wakatime.com/plugins<br><br>
 
-                # <strong>Step 2:</strong> Adapt your config<br>
-                $ vi ~/.wakatime.cfg<br>
-
+                # <strong>Step 2:</strong> Set your ~/.wakatime.cfg to this:<br><br>
                 <!-- https://github.com/muety/wakapi/issues/224#issuecomment-890855563 -->
-                # Set <em>api_url = <span class="with-url-inner">%s/api</span></em><br>
-                # Set <em>api_key = <span id="api-key-instruction">{{ .ApiKey }}</span></em><br><br>
+                [settings]<br>
+                api_url = <span class="with-url-inner">%s/api</span><br>
+                api_key = <span id="api-key-instruction">{{ .ApiKey }}</span><br><br>
 
                 # <strong>Step 3:</strong> Start coding and then check back here!
             </div>