Add: database config for ro and rw pools

Author: robalb

backend/.gitignore

@@ -1,5 +1,7 @@
 tmp/
 *.sqlite
+*.sqlite-shm
+*.sqlite-wal
 main
 stress/
 

backend/e2e/database_test.go

@@ -0,0 +1,79 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/robalb/morsechat/internal/db"
+)
+
+func TestDatabase(t *testing.T) {
+	t.Parallel()
+	ctx := context.Background()
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	t.Cleanup(cancel)
+
+
+	tempdb, cleandb, err := NewVolatileSqliteFile()
+	if err != nil {
+		t.Fatalf("Could not generate temporary db file")
+	}
+	t.Cleanup(cleandb)
+
+  dbWritePool, err := db.NewWritePool(tempdb, ctx)
+	if err != nil {
+		t.Fatalf("Failed to init database write pool: %v", err.Error())
+	}
+  dbReadPool, err := db.NewReadPool(tempdb, ctx)
+	if err != nil {
+		t.Fatalf("Failed to init database read pool: %v", err.Error())
+	}
+	err = db.ApplyMigrations(dbWritePool, ctx)
+	if err != nil {
+		t.Fatalf("Failed to apply database migrations: %v", err.Error())
+	}
+
+	readQueries := db.New(dbReadPool)
+	writeQueries := db.New(dbWritePool)
+
+	// perform a read query using the read pool
+  {
+    authors, err := readQueries.ListModerators(ctx)
+    if err != nil {
+      t.Fatalf("error query db: %v", err)
+    }
+    fmt.Println(authors)
+  }
+
+	// perform a write query using the read pool
+  {
+    _, err := readQueries.CreateUser(ctx, db.CreateUserParams{
+      Username:            "lorem",
+      Callsign:            "US121X",
+      Country:             "US",
+      RegistrationSession: "afakeuuidv4",
+    })
+    if err == nil {
+      t.Fatalf("a write query on a readonly db connection did not raise errors")
+    } else{
+      fmt.Printf("db raised err as expected: %v", err)
+    }
+  }
+
+	// perform a write query using the write pool
+  {
+    _, err := writeQueries.CreateUser(ctx, db.CreateUserParams{
+      Username:            "lorem",
+      Callsign:            "US121X",
+      Country:             "US",
+      RegistrationSession: "afakeuuidv4",
+    })
+    if err != nil {
+      t.Fatalf("db error writing with a write connection")
+    } 
+  }
+
+}
+

backend/internal/db/conn.go

@@ -4,14 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"fmt"
+	"strings"
 )
 
 //go:embed schema.sql
 var initscript string
 
-// global, hardcoded sqlite config
-var sqliteConfig string = "?_foreign_keys=true"
-
 // Temporary solution. We don't actually have migrations right now.
 // We just execute the content of schema.sql every time
 func ApplyMigrations(db *sql.DB, ctx context.Context) error {
@@ -26,6 +25,8 @@ func ApplyMigrations(db *sql.DB, ctx context.Context) error {
 // TODO: runtime pragmas, as seen here
 // https://github.com/mtlynch/picoshare/blob/master/store/sqlite/sqlite.go
 func NewTestConn(sqlitePath string, ctx context.Context) (*sql.DB, error) {
+  // global, hardcoded sqlite config
+  var sqliteConfig string = "?_foreign_keys=true"
 	db, err := sql.Open("sqlite3", sqlitePath+sqliteConfig)
 	if err != nil {
 		return nil, err
@@ -35,58 +36,35 @@ func NewTestConn(sqlitePath string, ctx context.Context) (*sql.DB, error) {
 }
 
 // Return a readonly connection to the sqlite database
-// TODO: proper config
 // TODO: ping()
 func NewReadPool(sqlitePath string, ctx context.Context) (*sql.DB, error) {
-	return sql.Open("sqlite3", sqlitePath+sqliteConfig)
+  // https://github.com/mattn/go-sqlite3/issues/1022#issuecomment-1067353980
+  // https://github.com/mattn/go-sqlite3?tab=readme-ov-file#connection-string
+  options := []string{
+    "_foreign_keys=true",
+    "_journal_mode=wal",
+    "_busy_timeout=5000",
+    "mode=ro",
+  }
+  connString := fmt.Sprintf("file:%s?%s", sqlitePath, strings.Join(options, "&"))
+	return sql.Open("sqlite3", connString)
 }
 
 // Return a read+write connection to the sqlite database
-// TODO: proper config
 func NewWritePool(sqlitePath string, ctx context.Context) (conn *sql.DB, err error) {
-	conn, err = sql.Open("sqlite3", sqlitePath+sqliteConfig)
+  // https://github.com/mattn/go-sqlite3/issues/1022#issuecomment-1067353980
+  // https://github.com/mattn/go-sqlite3?tab=readme-ov-file#connection-string
+  options := []string{
+    "_foreign_keys=true",
+    "_journal_mode=wal",
+    "_txlock=immediate",
+    "_busy_timeout=5000",
+    "mode=rwc",
+  }
+  connString := fmt.Sprintf("file:%s?%s", sqlitePath, strings.Join(options, "&"))
+	conn, err = sql.Open("sqlite3", connString)
 	//sqlite does not support cuncurrent write
-	//TODO: test performance implications
-	// conn.SetMaxOpenConns(1)
+	conn.SetMaxOpenConns(1)
 	return
 }
 
-// func SQLite(t *testing.T, migrations []string) (*sql.DB, func()) {
-// 	t.Helper()
-// 	// For each test, pick a new database name at random.
-// 	source, err := os.CreateTemp("", "sqltest_sqlite_")
-// 	if err != nil {
-// 		t.Fatal(err)
-// 	}
-// 	return CreateSQLiteDatabase(t, source.Name(), migrations)
-// }
-
-// func CreateSQLiteDatabase(t *testing.T, path string, migrations []string) (*sql.DB, func()) {
-// 	t.Helper()
-
-// 	t.Logf("open %s\n", path)
-// 	sdb, err := sql.Open("sqlite", path)
-// 	if err != nil {
-// 		t.Fatal(err)
-// 	}
-
-// 	files, err := sqlpath.Glob(migrations)
-// 	if err != nil {
-// 		t.Fatal(err)
-// 	}
-// 	for _, f := range files {
-// 		blob, err := os.ReadFile(f)
-// 		if err != nil {
-// 			t.Fatal(err)
-// 		}
-// 		if _, err := sdb.Exec(string(blob)); err != nil {
-// 			t.Fatalf("%s: %s", filepath.Base(f), err)
-// 		}
-// 	}
-
-// 	return sdb, func() {
-// 		if _, err := os.Stat(path); err == nil {
-// 			os.Remove(path)
-// 		}
-// 	}
-// }

backend/internal/db/conn_test.go

@@ -61,3 +61,4 @@ func TestConn(t *testing.T) {
 	}
 
 }
+

backend/internal/server/run.go

@@ -51,14 +51,14 @@ func Run(
 	// Init JWT auth
 	tokenAuth := jwtauth.New("HS256", config.SecretBytes, nil)
 	// Init db
-	dbReadPool, err := db.NewReadPool(config.SqlitePath, ctx)
+	dbWritePool, err := db.NewWritePool(config.SqlitePath, ctx)
 	if err != nil {
-		logger.Printf("Failed to init database read pool: %v", err.Error())
+		logger.Printf("Failed to init database write pool: %v", err.Error())
 		return err
 	}
-	dbWritePool, err := db.NewWritePool(config.SqlitePath, ctx)
+	dbReadPool, err := db.NewReadPool(config.SqlitePath, ctx)
 	if err != nil {
-		logger.Printf("Failed to init database write pool: %v", err.Error())
+		logger.Printf("Failed to init database read pool: %v", err.Error())
 		return err
 	}
 	err = db.ApplyMigrations(dbWritePool, ctx)