Automatic database optimization and performance tweaks (#3527)

* Automatic database optimize
* Tweak connection pooling

pkg/sqlite/database.go

@@ -2,7 +2,6 @@ package sqlite
 
 import (
 	"context"
-	"database/sql"
 	"embed"
 	"errors"
 	"fmt"
@@ -10,17 +9,29 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/fvbommel/sortorder"
 	"github.com/golang-migrate/migrate/v4"
 	sqlite3mig "github.com/golang-migrate/migrate/v4/database/sqlite3"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
 	"github.com/jmoiron/sqlx"
-	sqlite3 "github.com/mattn/go-sqlite3"
 
 	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/logger"
 )
 
+const (
+	// Number of database connections to use
+	// The same value is used for both the maximum and idle limit,
+	// to prevent opening connections on the fly which has a notieable performance penalty.
+	// Fewer connections use less memory, more connections increase performance,
+	// but have diminishing returns.
+	// 10 was found to be a good tradeoff.
+	dbConns = 10
+	// Idle connection timeout, in seconds
+	// Closes a connection after a period of inactivity, which saves on memory and
+	// causes the sqlite -wal and -shm files to be automatically deleted.
+	dbConnTimeout = 30
+)
+
 var appSchemaVersion uint = 43
 
 //go:embed migrations/*.sql
@@ -52,13 +63,6 @@ func (e *MismatchedSchemaVersionError) Error() string {
 	return fmt.Sprintf("schema version %d is incompatible with required schema version %d", e.CurrentSchemaVersion, e.RequiredSchemaVersion)
 }
 
-const sqlite3Driver = "sqlite3ex"
-
-func init() {
-	// register custom driver with regexp function
-	registerCustomDriver()
-}
-
 type Database struct {
 	File      *FileStore
 	Folder    *FolderStore
@@ -202,9 +206,9 @@ func (db *Database) open(disableForeignKeys bool) (*sqlx.DB, error) {
 	}
 
 	conn, err := sqlx.Open(sqlite3Driver, url)
-	conn.SetMaxOpenConns(25)
-	conn.SetMaxIdleConns(4)
-	conn.SetConnMaxLifetime(30 * time.Second)
+	conn.SetMaxOpenConns(dbConns)
+	conn.SetMaxIdleConns(dbConns)
+	conn.SetConnMaxIdleTime(dbConnTimeout * time.Second)
 	if err != nil {
 		return nil, fmt.Errorf("db.Open(): %w", err)
 	}
@@ -453,38 +457,3 @@ func (db *Database) runCustomMigration(ctx context.Context, fn customMigrationFu
 
 	return nil
 }
-
-func registerCustomDriver() {
-	sql.Register(sqlite3Driver,
-		&sqlite3.SQLiteDriver{
-			ConnectHook: func(conn *sqlite3.SQLiteConn) error {
-				funcs := map[string]interface{}{
-					"regexp":            regexFn,
-					"durationToTinyInt": durationToTinyIntFn,
-					"basename":          basenameFn,
-				}
-
-				for name, fn := range funcs {
-					if err := conn.RegisterFunc(name, fn, true); err != nil {
-						return fmt.Errorf("error registering function %s: %s", name, err.Error())
-					}
-				}
-
-				// COLLATE NATURAL_CS - Case sensitive natural sort
-				err := conn.RegisterCollation("NATURAL_CS", func(s string, s2 string) int {
-					if sortorder.NaturalLess(s, s2) {
-						return -1
-					} else {
-						return 1
-					}
-				})
-
-				if err != nil {
-					return fmt.Errorf("error registering natural sort collation: %v", err)
-				}
-
-				return nil
-			},
-		},
-	)
-}

pkg/sqlite/driver.go

@@ -0,0 +1,71 @@
+package sqlite
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+
+	"github.com/fvbommel/sortorder"
+	sqlite3 "github.com/mattn/go-sqlite3"
+)
+
+const sqlite3Driver = "sqlite3ex"
+
+func init() {
+	// register custom driver
+	sql.Register(sqlite3Driver, &CustomSQLiteDriver{})
+}
+
+type CustomSQLiteDriver struct{}
+
+type CustomSQLiteConn struct {
+	*sqlite3.SQLiteConn
+}
+
+func (d *CustomSQLiteDriver) Open(dsn string) (driver.Conn, error) {
+	sqlite3Driver := &sqlite3.SQLiteDriver{
+		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+			funcs := map[string]interface{}{
+				"regexp":            regexFn,
+				"durationToTinyInt": durationToTinyIntFn,
+				"basename":          basenameFn,
+			}
+
+			for name, fn := range funcs {
+				if err := conn.RegisterFunc(name, fn, true); err != nil {
+					return fmt.Errorf("error registering function %s: %v", name, err)
+				}
+			}
+
+			// COLLATE NATURAL_CS - Case sensitive natural sort
+			err := conn.RegisterCollation("NATURAL_CS", func(s string, s2 string) int {
+				if sortorder.NaturalLess(s, s2) {
+					return -1
+				} else {
+					return 1
+				}
+			})
+
+			if err != nil {
+				return fmt.Errorf("error registering natural sort collation: %v", err)
+			}
+
+			return nil
+		},
+	}
+
+	conn, err := sqlite3Driver.Open(dsn)
+	if err != nil {
+		return nil, err
+	}
+
+	return &CustomSQLiteConn{conn.(*sqlite3.SQLiteConn)}, nil
+}
+
+func (c *CustomSQLiteConn) Close() error {
+	conn := c.SQLiteConn
+
+	_, _ = conn.Exec("PRAGMA analysis_limit=1000; PRAGMA optimize;", []driver.Value{})
+
+	return conn.Close()
+}