sqlitemigrate patches for larger datasets

scripts/sqlite2mysql.go

@@ -4,10 +4,11 @@ package main
 Usage:
 ---
 1. Set up a MySQL instance (see docker_mysql.sh for example)
-2. Create config file (e.g. migrate.yml) as shown below
-3. go run sqlite2mysql.go -config migrate.yml
+2. Create config file (e.g. config.yml) as shown below
+3. go run sqlite2mysql.go -config config.yml
+4. For postgres check https://wiki.postgresql.org/wiki/Fixing_Sequences
 
-Example: migrate.yml
+Example: config.yml
 ---
 # SQLite
 source:
@@ -21,19 +22,22 @@ target:
   user:
   password:
   name:
+  dialect:
 */
 
 import (
 	"flag"
 	"fmt"
+	"log"
+	"os"
+
 	"github.com/jinzhu/configor"
 	"github.com/muety/wakapi/models"
 	"github.com/muety/wakapi/repositories"
 	"gorm.io/driver/mysql"
+	"gorm.io/driver/postgres"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
-	"log"
-	"os"
 )
 
 type config struct {
@@ -47,6 +51,7 @@ type dbConfig struct {
 	User     string
 	Password string
 	Name     string
+	Dialect  string `yaml:"-"`
 }
 
 var cfg *config
@@ -75,6 +80,20 @@ func init() {
 		dbSource = db
 	}
 
+	if cfg.Target.Dialect == "postgres" {
+		log.Println("attempting to open postgresql database as Target")
+		if db, err := gorm.Open(postgres.Open(fmt.Sprintf("user=%s password=%s host=%s port=%d dbname=%s sslmode=disable timezone=Europe/Berlin",
+			cfg.Target.User,
+			cfg.Target.Password,
+			cfg.Target.Host,
+			cfg.Target.Port,
+			cfg.Target.Name,
+		)), &gorm.Config{}); err != nil {
+			log.Fatalln(err)
+		} else {
+			dbTarget = db
+		}
+	} else {
 		log.Println("attempting to open mysql database as Target")
 		if db, err := gorm.Open(mysql.New(mysql.Config{
 			DriverName: "mysql",
@@ -93,6 +112,7 @@ func init() {
 			dbTarget = db
 		}
 	}
+}
 
 func destroy() {
 	if db, _ := dbSource.DB(); db != nil {
@@ -191,7 +211,25 @@ func main() {
 	log.Println("Migrating heartbeats ...")
 	if data, err := heartbeatSource.GetAll(); err == nil {
 		log.Printf("Got %d heartbeats loaded into memory. Batch-inserting them now ...\n", len(data))
-		if err := heartbeatTarget.InsertBatch(data); err != nil {
+
+		var slice = make([]*models.Heartbeat, len(data))
+		for i, heartbeat := range data {
+			heartbeat = heartbeat.Hashed()
+			log.Println(heartbeat.String())
+			slice[i] = heartbeat
+		}
+		left := 0
+		right := 100
+		size := len(slice)
+		for right < size {
+			log.Printf("Inserting batch from %d", left)
+			if err := heartbeatTarget.InsertBatch(slice[left:right]); err != nil {
+				log.Fatalln(err)
+			}
+			left += 100
+			right += 100
+		}
+		if err := heartbeatTarget.InsertBatch(slice[left:]); err != nil {
 			log.Fatalln(err)
 		}
 	} else {