package main /* Usage: --- 1. Set up a MySQL instance (see docker_mysql.sh for example) 2. Create config file (e.g. migrate.yml) as shown below 3. go run sqlite2mysql.go Example: migrate.yml --- # SQLite source: # Example: ../wakapi_db.db (relative to script path) name: # MySQL target: host: port: user: password: name: */ import ( "flag" "fmt" "github.com/jinzhu/configor" "github.com/muety/wakapi/models" "github.com/muety/wakapi/repositories" "gorm.io/driver/mysql" "gorm.io/driver/sqlite" "gorm.io/gorm" "log" "os" ) type config struct { Source dbConfig // sqlite Target dbConfig // mysql } type dbConfig struct { Host string Port uint User string Password string Name string } var cfg *config var dbSource, dbTarget *gorm.DB var cFlag *string func init() { cfg = &config{} if f := flag.Lookup("config"); f == nil { cFlag = flag.String("config", "sqlite2mysql.yml", "config file location") } else { ff := f.Value.(flag.Getter).Get().(string) cFlag = &ff } flag.Parse() if err := configor.New(&configor.Config{}).Load(cfg, mustConfigPath()); err != nil { log.Fatalln("failed to read config", err) } log.Println("attempting to open sqlite database as Source") if db, err := gorm.Open(sqlite.Open(cfg.Source.Name), &gorm.Config{}); err != nil { log.Fatalln(err) } else { dbSource = db } log.Println("attempting to open mysql database as Target") if db, err := gorm.Open(mysql.New(mysql.Config{ DriverName: "mysql", DSN: fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=true&loc=%s&sql_mode=ANSI_QUOTES", cfg.Target.User, cfg.Target.Password, cfg.Target.Host, cfg.Target.Port, cfg.Target.Name, "utf8mb4", "Local", ), }), &gorm.Config{}); err != nil { log.Fatalln(err) } else { dbTarget = db } } func destroy() { if db, _ := dbSource.DB(); db != nil { db.Close() } if db, _ := dbTarget.DB(); db != nil { db.Close() } } func main() { defer destroy() if err := createSchema(); err != nil { log.Fatalln(err) } keyValueSource := repositories.NewKeyValueRepository(dbSource) keyValueTarget := repositories.NewKeyValueRepository(dbTarget) userSource := repositories.NewUserRepository(dbSource) userTarget := repositories.NewUserRepository(dbTarget) languageMappingSource := repositories.NewLanguageMappingRepository(dbSource) languageMappingTarget := repositories.NewLanguageMappingRepository(dbTarget) aliasSource := repositories.NewAliasRepository(dbSource) aliasTarget := repositories.NewAliasRepository(dbTarget) summarySource := repositories.NewSummaryRepository(dbSource) summaryTarget := repositories.NewSummaryRepository(dbTarget) heartbeatSource := repositories.NewHeartbeatRepository(dbSource) heartbeatTarget := repositories.NewHeartbeatRepository(dbTarget) // TODO: things could be optimized through batch-inserts / inserts within a single transaction log.Println("Migrating key-value pairs ...") if data, err := keyValueSource.GetAll(); err == nil { for _, e := range data { if err := keyValueTarget.PutString(e); err != nil { log.Fatalln(err) } } } else { log.Fatalln(err) } log.Println("Migrating users ...") if data, err := userSource.GetAll(); err == nil { for _, e := range data { if _, _, err := userTarget.InsertOrGet(e); err != nil { log.Fatalln(err) } } } else { log.Fatalln(err) } log.Println("Migrating language mappings ...") if data, err := languageMappingSource.GetAll(); err == nil { for _, e := range data { e.ID = 0 if _, err := languageMappingTarget.Insert(e); err != nil { log.Fatalln(err) } } } else { log.Fatalln(err) } log.Println("Migrating aliases ...") if data, err := aliasSource.GetAll(); err == nil { for _, e := range data { e.ID = 0 if _, err := aliasTarget.Insert(e); err != nil { log.Fatalln(err) } } } else { log.Fatalln(err) } log.Println("Migrating summaries ...") if data, err := summarySource.GetAll(); err == nil { for _, e := range data { e.ID = 0 if err := summaryTarget.Insert(e); err != nil { log.Fatalln(err) } } } else { log.Fatalln(err) } // TODO: copy in mini-batches instead of loading all heartbeats into memory (potentially millions) log.Println("Migrating heartbeats ...") if data, err := heartbeatSource.GetAll(); err == nil { log.Printf("Got %d heartbeats loaded into memory. Batch-inserting them now ...\n", len(data)) if err := heartbeatTarget.InsertBatch(data); err != nil { log.Fatalln(err) } } else { log.Fatalln(err) } } func createSchema() error { if err := dbTarget.AutoMigrate(&models.User{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.KeyStringValue{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.Alias{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.Heartbeat{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.Summary{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.SummaryItem{}); err != nil { return err } if err := dbTarget.AutoMigrate(&models.LanguageMapping{}); err != nil { return err } return nil } func mustConfigPath() string { if _, err := os.Stat(*cFlag); err != nil { log.Fatalln("failed to find config file at", *cFlag) } return *cFlag }