diff --git a/pkg/database/custom_migrations.go b/pkg/database/custom_migrations.go new file mode 100644 index 000000000..d3c234548 --- /dev/null +++ b/pkg/database/custom_migrations.go @@ -0,0 +1,69 @@ +package database + +import ( + "database/sql" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/pkg/logger" +) + +func runCustomMigrations() error { + if err := createImagesChecksumIndex(); err != nil { + return err + } + + return nil +} + +func createImagesChecksumIndex() error { + return WithTxn(func(tx *sqlx.Tx) error { + row := tx.QueryRow("SELECT 1 AS found FROM sqlite_master WHERE type = 'index' AND name = 'images_checksum_unique'") + err := row.Err() + if err != nil && err != sql.ErrNoRows { + return err + } + + if err == nil { + var found bool + row.Scan(&found) + if found { + return nil + } + } + + _, err = tx.Exec("CREATE UNIQUE INDEX images_checksum_unique ON images (checksum)") + if err == nil { + _, err = tx.Exec("DROP INDEX IF EXISTS index_images_checksum") + if err != nil { + logger.Errorf("Failed to remove surrogate images.checksum index: %s", err) + } + logger.Info("Created unique constraint on images table") + return nil + } + + _, err = tx.Exec("CREATE INDEX IF NOT EXISTS index_images_checksum ON images (checksum)") + if err != nil { + logger.Errorf("Unable to create index on images.checksum: %s", err) + } + + var result []struct { + Checksum string `db:"checksum"` + } + + err = tx.Select(&result, "SELECT checksum FROM images GROUP BY checksum HAVING COUNT(1) > 1") + if err != nil && err != sql.ErrNoRows { + logger.Errorf("Unable to determine non-unique image checksums: %s", err) + return nil + } + + checksums := make([]string, len(result)) + for i, res := range result { + checksums[i] = res.Checksum + } + + logger.Warnf("The following duplicate image checksums have been found. Please remove the duplicates and restart. %s", strings.Join(checksums, ", ")) + + return nil + }) +} diff --git a/pkg/database/database.go b/pkg/database/database.go index f2aa58734..017a3497a 100644 --- a/pkg/database/database.go +++ b/pkg/database/database.go @@ -23,7 +23,7 @@ import ( var DB *sqlx.DB var WriteMu *sync.Mutex var dbPath string -var appSchemaVersion uint = 27 +var appSchemaVersion uint = 28 var databaseSchemaVersion uint var ( @@ -86,6 +86,10 @@ func Initialize(databasePath string) error { DB = open(databasePath, disableForeignKeys) WriteMu = &sync.Mutex{} + if err := runCustomMigrations(); err != nil { + return err + } + return nil } diff --git a/pkg/database/migrations/28_images_indexes.up.sql b/pkg/database/migrations/28_images_indexes.up.sql new file mode 100644 index 000000000..1fbb1cfe2 --- /dev/null +++ b/pkg/database/migrations/28_images_indexes.up.sql @@ -0,0 +1,3 @@ +DROP INDEX IF EXISTS `images_path_unique`; + +CREATE UNIQUE INDEX `images_path_unique` ON `images` (`path`); diff --git a/ui/v2.5/src/components/Changelog/versions/v0100.md b/ui/v2.5/src/components/Changelog/versions/v0100.md index cd6b2aa1b..a332998d3 100644 --- a/ui/v2.5/src/components/Changelog/versions/v0100.md +++ b/ui/v2.5/src/components/Changelog/versions/v0100.md @@ -1,3 +1,5 @@ +#### 💥 Note: Please check your logs after migrating to this release. A log warning will be generated on startup if duplicate image checksums exist in your system. Search for the images using the logged checksums, and remove the unwanted ones. + ### ✨ New Features * Added options to generate webp and static preview files for markers. ([#1604](https://github.com/stashapp/stash/pull/1604)) * Added sort by option for gallery rating. ([#1720](https://github.com/stashapp/stash/pull/1720)) @@ -10,6 +12,7 @@ * Support filtering Movies by Performers. ([#1675](https://github.com/stashapp/stash/pull/1675)) ### 🎨 Improvements +* Added missing image table indexes, resulting in a significant performance improvement. ([#1740](https://github.com/stashapp/stash/pull/1740)) * Support scraper script logging to specific log levels. ([#1648](https://github.com/stashapp/stash/pull/1648)) * Added sv-SE language option. ([#1691](https://github.com/stashapp/stash/pull/1691))