File storage rewrite (#2676)

* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes

pkg/sqlite/database.go

@@ -1,6 +1,7 @@
 package sqlite
 
 import (
+	"context"
 	"database/sql"
 	"embed"
 	"errors"
@@ -20,7 +21,7 @@ import (
 	"github.com/stashapp/stash/pkg/logger"
 )
 
-var appSchemaVersion uint = 31
+var appSchemaVersion uint = 32
 
 //go:embed migrations/*.sql
 var migrationsBox embed.FS
@@ -59,6 +60,12 @@ func init() {
 }
 
 type Database struct {
+	File    *FileStore
+	Folder  *FolderStore
+	Image   *ImageStore
+	Gallery *GalleryStore
+	Scene   *SceneStore
+
 	db     *sqlx.DB
 	dbPath string
 
@@ -67,6 +74,16 @@ type Database struct {
 	writeMu sync.Mutex
 }
 
+func NewDatabase() *Database {
+	return &Database{
+		File:    NewFileStore(),
+		Folder:  NewFolderStore(),
+		Image:   NewImageStore(),
+		Gallery: NewGalleryStore(),
+		Scene:   NewSceneStore(),
+	}
+}
+
 // Ready returns an error if the database is not ready to begin transactions.
 func (db *Database) Ready() error {
 	if db.db == nil {
@@ -124,10 +141,6 @@ func (db *Database) Open(dbPath string) error {
 		}
 	}
 
-	if err := db.runCustomMigrations(); err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -246,7 +259,7 @@ func (db *Database) Version() uint {
 func (db *Database) getMigrate() (*migrate.Migrate, error) {
 	migrations, err := iofs.New(migrationsBox, "migrations")
 	if err != nil {
-		panic(err.Error())
+		return nil, err
 	}
 
 	const disableForeignKeys = true
@@ -282,6 +295,8 @@ func (db *Database) getDatabaseSchemaVersion() (uint, error) {
 
 // Migrate the database
 func (db *Database) RunMigrations() error {
+	ctx := context.Background()
+
 	m, err := db.getMigrate()
 	if err != nil {
 		return err
@@ -292,10 +307,27 @@ func (db *Database) RunMigrations() error {
 	stepNumber := appSchemaVersion - databaseSchemaVersion
 	if stepNumber != 0 {
 		logger.Infof("Migrating database from version %d to %d", databaseSchemaVersion, appSchemaVersion)
-		err = m.Steps(int(stepNumber))
-		if err != nil {
-			// migration failed
-			return err
+
+		// run each migration individually, and run custom migrations as needed
+		var i uint = 1
+		for ; i <= stepNumber; i++ {
+			newVersion := databaseSchemaVersion + i
+
+			// run pre migrations as needed
+			if err := db.runCustomMigrations(ctx, preMigrations[newVersion]); err != nil {
+				return fmt.Errorf("running pre migrations for schema version %d: %w", newVersion, err)
+			}
+
+			err = m.Steps(1)
+			if err != nil {
+				// migration failed
+				return err
+			}
+
+			// run post migrations as needed
+			if err := db.runCustomMigrations(ctx, postMigrations[newVersion]); err != nil {
+				return fmt.Errorf("running post migrations for schema version %d: %w", newVersion, err)
+			}
 		}
 	}
 
@@ -319,6 +351,31 @@ func (db *Database) RunMigrations() error {
 	return nil
 }
 
+func (db *Database) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error {
+	for _, fn := range fns {
+		if err := db.runCustomMigration(ctx, fn); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (db *Database) runCustomMigration(ctx context.Context, fn customMigrationFunc) error {
+	const disableForeignKeys = false
+	d, err := db.open(disableForeignKeys)
+	if err != nil {
+		return err
+	}
+
+	defer d.Close()
+	if err := fn(ctx, d); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func registerCustomDriver() {
 	sql.Register(sqlite3Driver,
 		&sqlite3.SQLiteDriver{