From 91b4cb2c13c8ea98db45234ffba0fe29b6a7aa23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ferdinand=20M=C3=BCtsch?= Date: Fri, 18 Mar 2022 13:41:32 +0100 Subject: [PATCH] fix: explicit milliseconds precision of timestamp columns --- .../20220318_mysql_timestamp_precision.go | 41 +++++++++++++++++++ models/heartbeat.go | 4 +- scripts/clean_duplicates.sql | 12 ++++++ scripts/count_duplicates_by_user.sql | 7 +--- services/duration.go | 6 +++ services/duration_test.go | 2 +- 6 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 migrations/20220318_mysql_timestamp_precision.go create mode 100644 scripts/clean_duplicates.sql diff --git a/migrations/20220318_mysql_timestamp_precision.go b/migrations/20220318_mysql_timestamp_precision.go new file mode 100644 index 0000000..cc1c0b1 --- /dev/null +++ b/migrations/20220318_mysql_timestamp_precision.go @@ -0,0 +1,41 @@ +package migrations + +import ( + "github.com/emvi/logbuch" + "github.com/muety/wakapi/config" + "gorm.io/gorm" +) + +func init() { + const name = "20220318-mysql_timestamp_precision" + f := migrationFunc{ + name: name, + f: func(db *gorm.DB, cfg *config.Config) error { + if hasRun(name, db) { + return nil + } + + if cfg.Db.IsMySQL() { + logbuch.Info("altering heartbeats table, this may take a while (up to hours)") + + db.Exec("SET foreign_key_checks=0;") + db.Exec("SET unique_checks=0;") + if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `time` TIMESTAMP(3) NOT NULL").Error; err != nil { + return err + } + if err := db.Exec("ALTER TABLE heartbeats MODIFY COLUMN `created_at` TIMESTAMP(3) NOT NULL").Error; err != nil { + return err + } + db.Exec("SET foreign_key_checks=1;") + db.Exec("SET unique_checks=1;") + + logbuch.Info("migrated timestamp columns to millisecond precision") + } + + setHasRun(name, db) + return nil + }, + } + + registerPostMigration(f) +} diff --git a/models/heartbeat.go b/models/heartbeat.go index 98cb45d..71f8dc8 100644 --- a/models/heartbeat.go +++ b/models/heartbeat.go @@ -23,11 +23,11 @@ type Heartbeat struct { OperatingSystem string `json:"operating_system" gorm:"index:idx_operating_system" hash:"ignore"` // ignored because os might be parsed differently by wakatime Machine string `json:"machine" gorm:"index:idx_machine" hash:"ignore"` // ignored because wakatime api doesn't return machines currently UserAgent string `json:"user_agent" hash:"ignore" gorm:"type:varchar(255)"` - Time CustomTime `json:"time" gorm:"type:timestamp; index:idx_time,idx_time_user" swaggertype:"primitive,number"` + Time CustomTime `json:"time" gorm:"type:timestamp(3); index:idx_time,idx_time_user" swaggertype:"primitive,number"` Hash string `json:"-" gorm:"type:varchar(17); uniqueIndex"` Origin string `json:"-" hash:"ignore" gorm:"type:varchar(255)"` OriginId string `json:"-" hash:"ignore" gorm:"type:varchar(255)"` - CreatedAt CustomTime `json:"created_at" gorm:"type:timestamp" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt + CreatedAt CustomTime `json:"created_at" gorm:"type:timestamp(3)" swaggertype:"primitive,number" hash:"ignore"` // https://gorm.io/docs/conventions.html#CreatedAt } func (h *Heartbeat) Valid() bool { diff --git a/scripts/clean_duplicates.sql b/scripts/clean_duplicates.sql new file mode 100644 index 0000000..153d894 --- /dev/null +++ b/scripts/clean_duplicates.sql @@ -0,0 +1,12 @@ +DELETE t1 +FROM heartbeats t1 + INNER JOIN heartbeats t2 +WHERE t1.id < t2.id + AND t1.time = t2.time + AND t1.entity = t2.entity + AND t1.is_write = t2.is_write + AND t1.branch = t2.branch + AND t1.editor = t2.editor + AND t1.machine = t2.machine + AND t1.operating_system = t2.operating_system + AND t1.user_id = t2.user_id; \ No newline at end of file diff --git a/scripts/count_duplicates_by_user.sql b/scripts/count_duplicates_by_user.sql index 751a9e5..22e71dc 100644 --- a/scripts/count_duplicates_by_user.sql +++ b/scripts/count_duplicates_by_user.sql @@ -1,11 +1,8 @@ SELECT s2.user_id, sum(c) as count, total, (sum(c) / total) as ratio FROM ( - SELECT time, - user_id, - entity, - COUNT(time) as c + SELECT time, user_id, entity, is_write, branch, editor, machine, operating_system, COUNT(time) as c FROM heartbeats - GROUP BY time, user_id, entity + GROUP BY time, user_id, entity, is_write, branch, editor, machine, operating_system HAVING COUNT(time) > 1 ) s2 LEFT JOIN (SELECT user_id, count(id) AS total FROM heartbeats GROUP BY user_id) s3 ON s2.user_id = s3.user_id diff --git a/services/duration.go b/services/duration.go index 6dedfbe..1b40220 100644 --- a/services/duration.go +++ b/services/duration.go @@ -36,6 +36,8 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters * } // Aggregation + // the below logic is approximately equivalent to the SQL query at scripts/aggregate_durations.sql, + // but unfortunately we cannot use it, as it features mysql-specific functions (lag(), timediff(), ...) var count int var latest *models.Duration @@ -91,5 +93,9 @@ func (srv *DurationService) Get(from, to time.Time, user *models.User, filters * } } + if len(heartbeats) == 1 && len(durations) == 1 { + durations[0].Duration = HeartbeatDiffThreshold + } + return durations.Sorted(), nil } diff --git a/services/duration_test.go b/services/duration_test.go index 7f793f5..edddfea 100644 --- a/services/duration_test.go +++ b/services/duration_test.go @@ -171,7 +171,7 @@ func (suite *DurationServiceTestSuite) TestDurationService_Get() { assert.Equal(suite.T(), TestEditorGoland, durations[0].Editor) assert.Equal(suite.T(), TestEditorGoland, durations[1].Editor) assert.Equal(suite.T(), TestEditorVscode, durations[2].Editor) - assert.Equal(suite.T(), 2, durations[0].NumHeartbeats) + assert.Equal(suite.T(), 3, durations[0].NumHeartbeats) assert.Equal(suite.T(), 1, durations[1].NumHeartbeats) assert.Equal(suite.T(), 3, durations[2].NumHeartbeats) }