From ad704cef5ceddc965e60ab90e3767418d84d7bab Mon Sep 17 00:00:00 2001
From: Steven Tang <yc@steventang.net>
Date: Tue, 27 Dec 2022 14:39:32 +1100
Subject: [PATCH 1/5] test: migration testing for mysql/mariadb/postgres

---
 .github/workflows/ci.yml                      |  8 +++-
 .gitignore                                    |  4 +-
 main.go                                       |  1 +
 testing/config.mysql.yml                      | 38 +++++++++++++++++
 testing/config.postgres.yml                   | 39 +++++++++++++++++
 .../{config.testing.yml => config.sqlite.yml} |  0
 testing/docker-compose.yml                    | 42 +++++++++++++++++++
 testing/run_api_tests.sh                      | 36 +++++++++++++---
 8 files changed, 159 insertions(+), 9 deletions(-)
 create mode 100644 testing/config.mysql.yml
 create mode 100644 testing/config.postgres.yml
 rename testing/{config.testing.yml => config.sqlite.yml} (100%)
 create mode 100644 testing/docker-compose.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cd58d91..8c1d8dd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -106,6 +106,11 @@ jobs:
     name: Migration tests
     runs-on: ubuntu-latest
 
+    strategy:
+      fail-fast: false
+      matrix:
+        db: [sqlite, postgres, mysql, mariadb]
+
     steps:
     - name: Set up Go 1.x
       uses: actions/setup-go@v2
@@ -116,5 +121,4 @@ jobs:
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
-    - name: Run sqlite
-      run: ./testing/run_api_tests.sh sqlite --migration
+    - run: ./testing/run_api_tests.sh ${{ matrix.db }} --migration
diff --git a/.gitignore b/.gitignore
index 34c07e3..f99d391 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,7 +8,7 @@ build
 *.zip
 config*.yml
 !config.default.yml
-!testing/config.testing.yml
+!testing/config.*.yml
 pkged.go
 package-lock.json
-node_modules
\ No newline at end of file
+node_modules
diff --git a/main.go b/main.go
index f724c46..5c03342 100644
--- a/main.go
+++ b/main.go
@@ -132,6 +132,7 @@ func main() {
 
 	// Connect to database
 	var err error
+	logbuch.Info("starting with %s database", config.Db.Dialect)
 	db, err = gorm.Open(config.Db.GetDialector(), &gorm.Config{Logger: gormLogger}, conf.GetWakapiDBOpts(&config.Db))
 	if err != nil {
 		logbuch.Error(err.Error())
diff --git a/testing/config.mysql.yml b/testing/config.mysql.yml
new file mode 100644
index 0000000..f4b7ad9
--- /dev/null
+++ b/testing/config.mysql.yml
@@ -0,0 +1,38 @@
+env: production
+
+server:
+  listen_ipv4: 127.0.0.1
+  listen_ipv6:
+  tls_cert_path:
+  tls_key_path:
+  port: 3000
+  base_path: /
+  public_url: http://localhost:3000
+
+app:
+  aggregation_time: '02:15'
+  report_time_weekly: 'fri,18:00'
+  heartbeat_max_age: 87600h  # 10 years
+  inactive_days: 7
+  custom_languages:
+    vue: Vue
+    jsx: JSX
+    svelte: Svelte
+
+db:
+  host: 127.0.0.1
+  port: 3306
+  user: wakapi
+  password: wakapi
+  name: wakapi
+  dialect: mysql
+  max_conn: 2
+  ssl: false
+  automgirate_fail_silently: false
+
+security:
+  password_salt:
+  insecure_cookies: true
+  cookie_max_age: 172800
+  allow_signup: true
+  expose_metrics: true
diff --git a/testing/config.postgres.yml b/testing/config.postgres.yml
new file mode 100644
index 0000000..0b5c672
--- /dev/null
+++ b/testing/config.postgres.yml
@@ -0,0 +1,39 @@
+env: production
+
+server:
+  listen_ipv4: 127.0.0.1
+  listen_ipv6:
+  tls_cert_path:
+  tls_key_path:
+  port: 3000
+  base_path: /
+  public_url: http://localhost:3000
+
+app:
+  aggregation_time: '02:15'
+  report_time_weekly: 'fri,18:00'
+  heartbeat_max_age: 87600h  # 10 years
+  inactive_days: 7
+  custom_languages:
+    vue: Vue
+    jsx: JSX
+    svelte: Svelte
+
+db:
+  host: 127.0.0.1
+  port: 5432
+  user: wakapi
+  password: wakapi
+  name: wakapi
+  dialect: postgres
+  charset:
+  max_conn: 2
+  ssl: false
+  automgirate_fail_silently: false
+
+security:
+  password_salt:
+  insecure_cookies: true
+  cookie_max_age: 172800
+  allow_signup: true
+  expose_metrics: true
diff --git a/testing/config.testing.yml b/testing/config.sqlite.yml
similarity index 100%
rename from testing/config.testing.yml
rename to testing/config.sqlite.yml
diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml
new file mode 100644
index 0000000..5ee6f0c
--- /dev/null
+++ b/testing/docker-compose.yml
@@ -0,0 +1,42 @@
+version: '3.7'
+
+services:
+  postgres:
+    image: postgres:15
+    environment:
+      POSTGRES_USER: "wakapi"
+      POSTGRES_PASSWORD: "wakapi"
+      POSTGRES_DB: "wakapi"
+    ports:
+      - "5432:5432"
+    volumes:
+      - wakapi-postgres:/var/lib/postgresql/data
+
+  mysql:
+    image: mysql:8
+    environment:
+      MYSQL_USER: "wakapi"
+      MYSQL_PASSWORD: "wakapi"
+      MYSQL_DATABASE: "wakapi"
+      MYSQL_ROOT_PASSWORD: example
+    ports:
+      - "3306:3306"
+    volumes:
+      - wakapi-mysql:/var/lib/mysql
+
+  mariadb:
+    image: mariadb:10
+    environment:
+      MARIADB_USER: "wakapi"
+      MARIADB_PASSWORD: "wakapi"
+      MARIADB_DATABASE: "wakapi"
+      MARIADB_ROOT_PASSWORD: example
+    ports:
+      - "3306:3306"
+    volumes:
+      - wakapi-mariadb:/var/lib/mysql
+
+volumes:
+  wakapi-postgres: {}
+  wakapi-mysql: {}
+  wakapi-mariadb: {}
diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index 447c6fb..e82bc7c 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -10,7 +10,6 @@ for i in "$@"; do
     case $i in
         --migration)
             MIGRATION=1
-            shift
             ;;
     esac
 done
@@ -37,6 +36,24 @@ fi
 
 # Initialise test data
 case $1 in
+    postgres|mysql|mariadb)
+    docker compose -f "$script_dir/docker-compose.yml" down
+    docker volume rm "testing_wakapi-$1"
+
+    docker_down=1
+    docker compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
+    if [ "$1" == "mariadb" ]; then
+        config="config.mysql.yml"
+    else
+        config="config.$1.yml"
+    fi
+    sleep 5
+
+    if [ "$1" == "mysql" ]; then
+        sleep 10
+    fi
+    ;;
+
     sqlite|*)
     rm -f wakapi_testing.db
 
@@ -46,7 +63,7 @@ case $1 in
     echo "Importing seed data ..."
     sqlite3 wakapi_testing.db < data.sql
 
-    config="config.testing.yml"
+    config="config.sqlite.yml"
     ;;
 esac
 
@@ -69,13 +86,18 @@ wait_for_wakapi () {
 
 # Run tests
 echo "Running Wakapi testing instance in background ..."
+echo "Configuration file: $config"
 "$initial_run_exe" -config "$config" &
 pid=$!
 wait_for_wakapi
 
-echo "Running test collection ..."
-newman run "wakapi_api_tests.postman_collection.json"
-exit_code=$?
+if [ "$1" == "sqlite" ]; then
+    echo "Running test collection ..."
+    newman run "wakapi_api_tests.postman_collection.json"
+    exit_code=$?
+else
+    exit_code=0
+fi
 
 echo "Shutting down Wakapi ..."
 kill -TERM $pid
@@ -91,5 +113,9 @@ if [[ $MIGRATION -eq 1 ]]; then
     kill -TERM $pid
 fi
 
+if [ "$docker_down" -eq 1 ]; then
+    docker compose -f "$script_dir/docker-compose.yml" down
+fi
+
 echo "Exiting with status $exit_code"
 exit $exit_code

From 03af194385acb55c6ceab55686532fd5cbe220ee Mon Sep 17 00:00:00 2001
From: Steven Tang <yc@steventang.net>
Date: Tue, 27 Dec 2022 15:35:11 +1100
Subject: [PATCH 2/5] test: more efficient database ready detection

---
 testing/docker-compose.yml |  9 +++------
 testing/run_api_tests.sh   | 15 ++++++++++++---
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml
index 5ee6f0c..3c72f98 100644
--- a/testing/docker-compose.yml
+++ b/testing/docker-compose.yml
@@ -7,8 +7,7 @@ services:
       POSTGRES_USER: "wakapi"
       POSTGRES_PASSWORD: "wakapi"
       POSTGRES_DB: "wakapi"
-    ports:
-      - "5432:5432"
+    network_mode: host
     volumes:
       - wakapi-postgres:/var/lib/postgresql/data
 
@@ -19,8 +18,7 @@ services:
       MYSQL_PASSWORD: "wakapi"
       MYSQL_DATABASE: "wakapi"
       MYSQL_ROOT_PASSWORD: example
-    ports:
-      - "3306:3306"
+    network_mode: host
     volumes:
       - wakapi-mysql:/var/lib/mysql
 
@@ -31,8 +29,7 @@ services:
       MARIADB_PASSWORD: "wakapi"
       MARIADB_DATABASE: "wakapi"
       MARIADB_ROOT_PASSWORD: example
-    ports:
-      - "3306:3306"
+    network_mode: host
     volumes:
       - wakapi-mariadb:/var/lib/mysql
 
diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index e82bc7c..902dc9a 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -47,11 +47,20 @@ case $1 in
     else
         config="config.$1.yml"
     fi
-    sleep 5
 
-    if [ "$1" == "mysql" ]; then
-        sleep 10
+    db_port=0
+    if [ "$1" == "postgres" ]; then
+        db_port=5432
+    else
+        db_port=3306
     fi
+
+    for _ in $(seq 0 30); do
+        if netstat -tulpn 2>/dev/null | grep "LISTEN" | tr -s ' ' | cut -d' ' -f4 | grep -E ":$db_port$" > /dev/null; then
+            break
+        fi
+        sleep 1
+    done
     ;;
 
     sqlite|*)

From 497046d0a4cdee20a65a8352f7c69e28ed02cb52 Mon Sep 17 00:00:00 2001
From: Steven Tang <yc@steventang.net>
Date: Tue, 27 Dec 2022 20:37:53 +1100
Subject: [PATCH 3/5] test: address pr comments

---
 testing/config.mysql.yml    |  2 +-
 testing/config.postgres.yml |  2 +-
 testing/docker-compose.yml  |  3 +++
 testing/run_api_tests.sh    | 22 ++++++++++++++--------
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/testing/config.mysql.yml b/testing/config.mysql.yml
index f4b7ad9..d382160 100644
--- a/testing/config.mysql.yml
+++ b/testing/config.mysql.yml
@@ -21,7 +21,7 @@ app:
 
 db:
   host: 127.0.0.1
-  port: 3306
+  port: 53306
   user: wakapi
   password: wakapi
   name: wakapi
diff --git a/testing/config.postgres.yml b/testing/config.postgres.yml
index 0b5c672..fec4144 100644
--- a/testing/config.postgres.yml
+++ b/testing/config.postgres.yml
@@ -21,7 +21,7 @@ app:
 
 db:
   host: 127.0.0.1
-  port: 5432
+  port: 55432
   user: wakapi
   password: wakapi
   name: wakapi
diff --git a/testing/docker-compose.yml b/testing/docker-compose.yml
index 3c72f98..77cab91 100644
--- a/testing/docker-compose.yml
+++ b/testing/docker-compose.yml
@@ -7,6 +7,7 @@ services:
       POSTGRES_USER: "wakapi"
       POSTGRES_PASSWORD: "wakapi"
       POSTGRES_DB: "wakapi"
+      PGPORT: 55432
     network_mode: host
     volumes:
       - wakapi-postgres:/var/lib/postgresql/data
@@ -14,6 +15,7 @@ services:
   mysql:
     image: mysql:8
     environment:
+      MYSQL_TCP_PORT: 53306
       MYSQL_USER: "wakapi"
       MYSQL_PASSWORD: "wakapi"
       MYSQL_DATABASE: "wakapi"
@@ -25,6 +27,7 @@ services:
   mariadb:
     image: mariadb:10
     environment:
+      MYSQL_TCP_PORT: 53306
       MARIADB_USER: "wakapi"
       MARIADB_PASSWORD: "wakapi"
       MARIADB_DATABASE: "wakapi"
diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index 902dc9a..5838051 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -34,14 +34,24 @@ if [[ $MIGRATION -eq 1 ]]; then
     echo "Running tests with release version"
 fi
 
+cleanup() {
+    if [ -n "$pid" ] && ps -p "$pid" > /dev/null; then
+        kill -TERM "$pid"
+    fi
+    if [ "${docker_down-0}" -eq 1 ]; then
+        docker-compose -f "$script_dir/docker-compose.yml" down
+    fi
+}
+trap cleanup EXIT
+
 # Initialise test data
 case $1 in
     postgres|mysql|mariadb)
-    docker compose -f "$script_dir/docker-compose.yml" down
+    docker-compose -f "$script_dir/docker-compose.yml" down
     docker volume rm "testing_wakapi-$1"
 
     docker_down=1
-    docker compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
+    docker-compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
     if [ "$1" == "mariadb" ]; then
         config="config.mysql.yml"
     else
@@ -50,9 +60,9 @@ case $1 in
 
     db_port=0
     if [ "$1" == "postgres" ]; then
-        db_port=5432
+        db_port=55432
     else
-        db_port=3306
+        db_port=53306
     fi
 
     for _ in $(seq 0 30); do
@@ -122,9 +132,5 @@ if [[ $MIGRATION -eq 1 ]]; then
     kill -TERM $pid
 fi
 
-if [ "$docker_down" -eq 1 ]; then
-    docker compose -f "$script_dir/docker-compose.yml" down
-fi
-
 echo "Exiting with status $exit_code"
 exit $exit_code

From 9cce0ac2e182d2bb6434b528653424e7429ec89a Mon Sep 17 00:00:00 2001
From: Steven Tang <yc@steventang.net>
Date: Tue, 27 Dec 2022 20:42:55 +1100
Subject: [PATCH 4/5] test: revert to docker compose

---
 testing/run_api_tests.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index 5838051..38b22c7 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -39,7 +39,7 @@ cleanup() {
         kill -TERM "$pid"
     fi
     if [ "${docker_down-0}" -eq 1 ]; then
-        docker-compose -f "$script_dir/docker-compose.yml" down
+        docker compose -f "$script_dir/docker-compose.yml" down
     fi
 }
 trap cleanup EXIT
@@ -47,11 +47,11 @@ trap cleanup EXIT
 # Initialise test data
 case $1 in
     postgres|mysql|mariadb)
-    docker-compose -f "$script_dir/docker-compose.yml" down
+    docker compose -f "$script_dir/docker-compose.yml" down
     docker volume rm "testing_wakapi-$1"
 
     docker_down=1
-    docker-compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
+    docker compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
     if [ "$1" == "mariadb" ]; then
         config="config.mysql.yml"
     else

From f46f24f0bed0b6f37260e1a412eeeed166dd840e Mon Sep 17 00:00:00 2001
From: Steven Tang <yc@steventang.net>
Date: Tue, 27 Dec 2022 20:58:17 +1100
Subject: [PATCH 5/5] test: ref the config conditional

---
 testing/run_api_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/run_api_tests.sh b/testing/run_api_tests.sh
index 38b22c7..4b2751b 100755
--- a/testing/run_api_tests.sh
+++ b/testing/run_api_tests.sh
@@ -52,10 +52,10 @@ case $1 in
 
     docker_down=1
     docker compose -f "$script_dir/docker-compose.yml" up --wait -d "$1"
+
+    config="config.$1.yml"
     if [ "$1" == "mariadb" ]; then
         config="config.mysql.yml"
-    else
-        config="config.$1.yml"
     fi
 
     db_port=0