Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 12:24:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

File storage rewrite (#2676)

Browse Source 
* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes
This commit is contained in:
WithoutPants
2022-07-13 16:30:54 +10:00
parent 30877c75fb
commit 5495d72849
359 changed files with 43690 additions and 16000 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
pkg/sqlite/custom_migrations.go
View File

@@ -2,78 +2,23 @@ package sqlite
import (
"context"
"database/sql"
"errors"
"fmt"
"strings"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/txn"
"github.com/jmoiron/sqlx"
)
func (db *Database) runCustomMigrations() error {
if err := db.createImagesChecksumIndex(); err != nil {
return err
}
type customMigrationFunc func(ctx context.Context, db *sqlx.DB) error
return nil
func RegisterPostMigration(schemaVersion uint, fn customMigrationFunc) {
v := postMigrations[schemaVersion]
v = append(v, fn)
postMigrations[schemaVersion] = v
}
func (db *Database) createImagesChecksumIndex() error {
return txn.WithTxn(context.Background(), db, func(ctx context.Context) error {
tx, err := getTx(ctx)
if err != nil {
return err
}
row := tx.QueryRow("SELECT 1 AS found FROM sqlite_master WHERE type = 'index' AND name = 'images_checksum_unique'")
err = row.Err()
if err != nil && !errors.Is(err, sql.ErrNoRows) {
return err
}
if err == nil {
var found bool
if err := row.Scan(&found); err != nil && err != sql.ErrNoRows {
return fmt.Errorf("error while scanning for index: %w", err)
}
if found {
return nil
}
}
_, err = tx.Exec("CREATE UNIQUE INDEX images_checksum_unique ON images (checksum)")
if err == nil {
_, err = tx.Exec("DROP INDEX IF EXISTS index_images_checksum")
if err != nil {
logger.Errorf("Failed to remove surrogate images.checksum index: %s", err)
}
logger.Info("Created unique constraint on images table")
return nil
}
_, err = tx.Exec("CREATE INDEX IF NOT EXISTS index_images_checksum ON images (checksum)")
if err != nil {
logger.Errorf("Unable to create index on images.checksum: %s", err)
}
var result []struct {
Checksum string `db:"checksum"`
}
err = tx.Select(&result, "SELECT checksum FROM images GROUP BY checksum HAVING COUNT(1) > 1")
if err != nil && !errors.Is(err, sql.ErrNoRows) {
logger.Errorf("Unable to determine non-unique image checksums: %s", err)
return nil
}
checksums := make([]string, len(result))
for i, res := range result {
checksums[i] = res.Checksum
}
logger.Warnf("The following duplicate image checksums have been found. Please remove the duplicates and restart. %s", strings.Join(checksums, ", "))
return nil
})
func RegisterPreMigration(schemaVersion uint, fn customMigrationFunc) {
v := preMigrations[schemaVersion]
v = append(v, fn)
preMigrations[schemaVersion] = v
}
var postMigrations = make(map[uint][]customMigrationFunc)
var preMigrations = make(map[uint][]customMigrationFunc)