fix: mysql character encoding (resolve #131)

README.md

@@ -172,7 +172,8 @@ You can specify configuration options either via a config file (default: `config
 | `db.user`                   | `WAKAPI_DB_USER`            | -              | Database user                                                       |
 | `db.password`               | `WAKAPI_DB_PASSWORD`        | -              | Database password                                                   |
 | `db.name`                   | `WAKAPI_DB_NAME`            | `wakapi_db.db` | Database name                                                       |
-| `db.dialect`                | `WAKAPI_DB_TYPE`            | `sqlite3`      | Database type (one of `sqlite3`, `mysql`, `postgres`, `cockroach`)                     |
+| `db.dialect`                | `WAKAPI_DB_TYPE`            | `sqlite3`      | Database type (one of `sqlite3`, `mysql`, `postgres`, `cockroach`)  |
+| `db.charset`                | `WAKAPI_DB_CHARSET`         | `utf8mb4`      | Database connection charset (for MySQL only)                        |
 | `db.max_conn`               | `WAKAPI_DB_MAX_CONNECTIONS` | `2`            | Maximum number of database connections                              |
 | `db.ssl`                    | `WAKAPI_DB_SSL`             | `false`        | Whether to use TLS encryption for database connection (Postgres and CockroachDB only) |
 

config.default.yml

@@ -22,6 +22,7 @@ db:
   password:                 # leave blank when using sqlite3
   name: wakapi_db.db        # database name for mysql / postgres or file path for sqlite (e.g. /tmp/wakapi.db)
   dialect: sqlite3          # mysql, postgres, sqlite3
+  charset: utf8mb4          # only used for mysql connections
   max_conn: 2               # maximum number of concurrent connections to maintain
   ssl: false                # whether to use tls for db connection (must be true for cockroachdb) (ignored for mysql and sqlite)
 

config/config.go

@@ -77,6 +77,7 @@ type dbConfig struct {
 	Password string `env:"WAKAPI_DB_PASSWORD"`
 	Name     string `default:"wakapi_db.db" env:"WAKAPI_DB_NAME"`
 	Dialect  string `yaml:"-"`
+	Charset  string `default:"utf8mb4" env:"WAKAPI_DB_CHARSET"`
 	Type     string `yaml:"dialect" default:"sqlite3" env:"WAKAPI_DB_TYPE"`
 	MaxConn  uint   `yaml:"max_conn" default:"2" env:"WAKAPI_DB_MAX_CONNECTIONS"`
 	Ssl      bool   `default:"false" env:"WAKAPI_DB_SSL"`
@@ -181,12 +182,13 @@ func (c *dbConfig) GetDialector() gorm.Dialector {
 
 func mysqlConnectionString(config *dbConfig) string {
 	//location, _ := time.LoadLocation("Local")
-	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8&parseTime=true&loc=%s&sql_mode=ANSI_QUOTES",
+	return fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=%s&parseTime=true&loc=%s&sql_mode=ANSI_QUOTES",
 		config.User,
 		config.Password,
 		config.Host,
 		config.Port,
 		config.Name,
+		config.Charset,
 		"Local",
 	)
 }

main.go

@@ -171,10 +171,7 @@ func main() {
 
 	// Globally used middlewares
 	recoveryMiddleware := handlers.RecoveryHandler()
-	loggingMiddleware := middlewares.NewLoggingMiddleware(
+	loggingMiddleware := middlewares.NewLoggingMiddleware(logbuch.Info, []string{"/assets"})
-		log.New(os.Stdout, "", log.LstdFlags),
-		[]string{"/assets"},
-	)
 
 	// Router configs
 	router.Use(loggingMiddleware, recoveryMiddleware)

middlewares/logging.go

@@ -4,23 +4,24 @@ package middlewares
 
 import (
 	"io"
-	"log"
 	"net/http"
 	"strings"
 	"time"
 )
 
+type logFunc func(string, ...interface{})
+
 type LoggingMiddleware struct {
 	handler         http.Handler
-	output          *log.Logger
+	logFunc         logFunc
 	excludePrefixes []string
 }
 
-func NewLoggingMiddleware(output *log.Logger, excludePrefixes []string) func(http.Handler) http.Handler {
+func NewLoggingMiddleware(logFunc logFunc, excludePrefixes []string) func(http.Handler) http.Handler {
 	return func(h http.Handler) http.Handler {
 		return &LoggingMiddleware{
 			handler:         h,
-			output:          output,
+			logFunc:         logFunc,
 			excludePrefixes: excludePrefixes,
 		}
 	}
@@ -41,9 +42,8 @@ func (lg *LoggingMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	lg.output.Printf(
+	lg.logFunc(
-		"%v status=%d, method=%s, uri=%s, duration=%v, bytes=%d, addr=%s\n",
+		"[request] status=%d, method=%s, uri=%s, duration=%v, bytes=%d, addr=%s",
-		time.Now().Format(time.RFC3339Nano),
 		ww.Status(),
 		r.Method,
 		r.URL.String(),

migrations/20210206_drop_badges_column_add_sharing_flags.go

@@ -37,11 +37,15 @@ func init() {
 				return err
 			}
 
+			if cfg.Db.Dialect == config.SQLDialectSqlite {
+				logbuch.Info("not attempting to drop column 'badges_enabled' on sqlite")
+				return nil
+			}
+
 			if err := migrator.DropColumn(&models.User{}, "badges_enabled"); err != nil {
 				return err
-			} else {
-				logbuch.Info("dropped column 'badges_enabled' after substituting it by sharing indicators")
 			}
+			logbuch.Info("dropped column 'badges_enabled' after substituting it by sharing indicators")
 
 			return nil
 		},

scripts/push_sample_data.py

@@ -85,7 +85,7 @@ def post_data_sync(data: List[Heartbeat], url: str, api_key: str):
 
 
 def randomword(length: int) -> str:
-    letters = string.ascii_lowercase
+    letters = string.ascii_lowercase + 'äöü💩'  # test utf8 and utf8mb4 characters as well
     return ''.join(random.choice(letters) for _ in range(length))
 
 

version.txt

@@ -1 +1 @@
-1.24.1
+1.24.4