From 7cff71c35f0334c36a0d75872cc7757346cac141 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Fri, 17 Mar 2023 10:52:49 +1100 Subject: [PATCH] Add filesystem based blob storage (#3187) * Refactor transaction hooks. Add preCommit * Add BlobStore * Use blobStore for tag images * Use blobStore for studio images * Use blobStore for performer images * Use blobStore for scene covers * Don't generate screenshots in legacy directory * Run post-hooks outside original transaction * Use blobStore for movie images * Remove unnecessary DestroyImage methods * Add missing filter for scene cover * Add covers to generate options * Add generate cover option to UI * Add screenshot migration * Delete thumb files as part of screenshot migration --- gqlgen.yml | 2 + graphql/documents/data/config.graphql | 4 + graphql/documents/mutations/migration.graphql | 7 + graphql/schema/schema.graphql | 5 + graphql/schema/types/config.graphql | 17 + graphql/schema/types/metadata.graphql | 10 +- graphql/schema/types/migration.graphql | 11 + internal/api/resolver_mutation_configure.go | 27 +- internal/api/resolver_mutation_migrate.go | 40 ++ internal/api/resolver_mutation_movie.go | 42 +- internal/api/resolver_mutation_scene.go | 9 - internal/api/resolver_mutation_stash_box.go | 2 +- internal/api/resolver_mutation_studio.go | 7 +- internal/api/resolver_mutation_tag.go | 7 +- internal/api/resolver_query_configuration.go | 2 + internal/api/routes_movie.go | 10 +- internal/api/routes_performer.go | 5 +- internal/api/routes_studio.go | 5 +- internal/api/routes_tag.go | 5 +- internal/identify/identify.go | 3 +- internal/manager/config/config.go | 25 ++ internal/manager/config/enums.go | 50 +++ internal/manager/config/tasks.go | 2 + internal/manager/manager.go | 57 ++- internal/manager/manager_tasks.go | 10 +- internal/manager/repository.go | 2 - internal/manager/running_streams.go | 30 +- internal/manager/task/migrate_blobs.go | 129 ++++++ .../manager/task/migrate_scene_screenshots.go | 135 +++++++ internal/manager/task_generate.go | 66 +-- internal/manager/task_generate_screenshot.go | 69 ++-- internal/manager/task_identify.go | 8 +- internal/manager/task_scan.go | 43 +- pkg/file/clean.go | 4 +- pkg/file/delete.go | 8 +- pkg/file/fs.go | 20 + pkg/file/scan.go | 3 +- pkg/image/scan.go | 4 +- pkg/models/generate.go | 19 +- pkg/models/mocks/MovieReaderWriter.go | 56 +-- pkg/models/mocks/SceneReaderWriter.go | 35 +- pkg/models/mocks/StudioReaderWriter.go | 14 - pkg/models/mocks/TagReaderWriter.go | 14 - pkg/models/model_movie.go | 4 + pkg/models/model_studio.go | 2 + pkg/models/model_tag.go | 14 +- pkg/models/movie.go | 4 +- pkg/models/paths/paths.go | 5 +- pkg/models/paths/paths_scenes.go | 6 +- pkg/models/scene.go | 2 +- pkg/models/studio.go | 1 - pkg/models/tag.go | 1 - pkg/movie/import.go | 17 +- pkg/movie/import_test.go | 5 +- pkg/plugin/plugins.go | 3 +- pkg/scene/create.go | 14 - pkg/scene/delete.go | 12 - pkg/scene/generate/generator.go | 23 +- pkg/scene/generate/screenshot.go | 53 +-- pkg/scene/merge.go | 4 +- pkg/scene/migrate_hash.go | 8 - pkg/scene/migrate_screenshots.go | 143 +++++++ pkg/scene/scan.go | 13 +- pkg/scene/screenshot.go | 103 ----- pkg/scene/service.go | 1 + pkg/scene/update.go | 6 +- pkg/scene/update_test.go | 8 +- pkg/scraper/stashbox/stash_box.go | 2 +- pkg/sqlite/anonymise.go | 18 +- pkg/sqlite/blob.go | 382 ++++++++++++++++++ pkg/sqlite/blob/fs.go | 108 +++++ pkg/sqlite/blob_migrate.go | 116 ++++++ pkg/sqlite/blob_test.go | 45 +++ pkg/sqlite/database.go | 25 +- pkg/sqlite/migrations/45_blobs.up.sql | 19 + pkg/sqlite/migrations/45_postmigrate.go | 286 +++++++++++++ pkg/sqlite/movies.go | 75 ++-- pkg/sqlite/movies_test.go | 182 +++++---- pkg/sqlite/performer.go | 38 +- pkg/sqlite/performer_test.go | 21 +- pkg/sqlite/repository.go | 23 -- pkg/sqlite/scene.go | 42 +- pkg/sqlite/scene_test.go | 48 +-- pkg/sqlite/setup_test.go | 18 +- pkg/sqlite/sql.go | 25 -- pkg/sqlite/studio.go | 55 +-- pkg/sqlite/studio_test.go | 115 ++---- pkg/sqlite/tables.go | 7 + pkg/sqlite/tag.go | 67 ++- pkg/sqlite/tag_test.go | 101 +---- pkg/sqlite/transaction.go | 6 +- pkg/txn/hooks.go | 53 ++- pkg/txn/transaction.go | 48 ++- .../Settings/SettingsSystemPanel.tsx | 54 ++- .../Settings/Tasks/DataManagementTasks.tsx | 112 +++++ .../Settings/Tasks/GenerateOptions.tsx | 6 + .../Settings/Tasks/LibraryTasks.tsx | 1 + .../components/Settings/Tasks/ScanOptions.tsx | 7 + ui/v2.5/src/components/Setup/Setup.tsx | 83 ++++ ui/v2.5/src/core/StashService.ts | 14 + ui/v2.5/src/docs/en/Changelog/v0200.md | 13 + ui/v2.5/src/docs/en/ReleaseNotes/index.ts | 2 +- ui/v2.5/src/docs/en/ReleaseNotes/v0200.md | 8 +- ui/v2.5/src/locales/en-GB.json | 32 ++ .../models/list-filter/criteria/is-missing.ts | 1 + 105 files changed, 2647 insertions(+), 1086 deletions(-) create mode 100644 graphql/documents/mutations/migration.graphql create mode 100644 graphql/schema/types/migration.graphql create mode 100644 internal/api/resolver_mutation_migrate.go create mode 100644 internal/manager/config/enums.go create mode 100644 internal/manager/task/migrate_blobs.go create mode 100644 internal/manager/task/migrate_scene_screenshots.go create mode 100644 pkg/scene/migrate_screenshots.go delete mode 100644 pkg/scene/screenshot.go create mode 100644 pkg/sqlite/blob.go create mode 100644 pkg/sqlite/blob/fs.go create mode 100644 pkg/sqlite/blob_migrate.go create mode 100644 pkg/sqlite/blob_test.go create mode 100644 pkg/sqlite/migrations/45_blobs.up.sql create mode 100644 pkg/sqlite/migrations/45_postmigrate.go diff --git a/gqlgen.yml b/gqlgen.yml index 9e419a002..5be8c743a 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -34,6 +34,8 @@ models: title: resolver: true # autobind on config causes generation issues + BlobsStorageType: + model: github.com/stashapp/stash/internal/manager/config.BlobsStorageType StashConfig: model: github.com/stashapp/stash/internal/manager/config.StashConfig StashConfigInput: diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 40c8e9228..173a7948e 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -10,6 +10,8 @@ fragment ConfigGeneralData on ConfigGeneralResult { metadataPath scrapersPath cachePath + blobsPath + blobsStorage calculateMD5 videoFileNamingAlgorithm parallelTasks @@ -131,6 +133,7 @@ fragment ConfigDefaultSettingsData on ConfigDefaultSettingsResult { scan { useFileMetadata stripFileExtension + scanGenerateCovers scanGeneratePreviews scanGenerateImagePreviews scanGenerateSprites @@ -159,6 +162,7 @@ fragment ConfigDefaultSettingsData on ConfigDefaultSettingsResult { } generate { + covers sprites previews imagePreviews diff --git a/graphql/documents/mutations/migration.graphql b/graphql/documents/mutations/migration.graphql new file mode 100644 index 000000000..edf483276 --- /dev/null +++ b/graphql/documents/mutations/migration.graphql @@ -0,0 +1,7 @@ +mutation MigrateSceneScreenshots($input: MigrateSceneScreenshotsInput!) { + migrateSceneScreenshots(input: $input) +} + +mutation MigrateBlobs($input: MigrateBlobsInput!) { + migrateBlobs(input: $input) +} \ No newline at end of file diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index 85de2d826..87b0a916f 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -287,8 +287,13 @@ type Mutation { metadataClean(input: CleanMetadataInput!): ID! """Identifies scenes using scrapers. Returns the job ID""" metadataIdentify(input: IdentifyMetadataInput!): ID! + """Migrate generated files for the current hash naming""" migrateHashNaming: ID! + """Migrates legacy scene screenshot files into the blob storage""" + migrateSceneScreenshots(input: MigrateSceneScreenshotsInput!): ID! + """Migrates blobs from the old storage system to the current one""" + migrateBlobs(input: MigrateBlobsInput!): ID! """Anonymise the database in a separate file. Optionally returns a link to download the database file""" anonymiseDatabase(input: AnonymiseDatabaseInput!): String diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 9e7ee529e..df0aba092 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -8,6 +8,8 @@ input SetupInput { generatedLocation: String! """Empty to indicate default""" cacheLocation: String! + """Empty to indicate database storage for blobs""" + blobsLocation: String! } enum StreamingResolutionEnum { @@ -34,6 +36,13 @@ enum HashAlgorithm { "oshash", OSHASH } +enum BlobsStorageType { + # blobs are stored in the database + "Database", DATABASE + # blobs are stored in the filesystem under the configured blobs directory + "Filesystem", FILESYSTEM +} + input ConfigGeneralInput { """Array of file paths to content""" stashes: [StashConfigInput!] @@ -49,6 +58,10 @@ input ConfigGeneralInput { scrapersPath: String """Path to cache""" cachePath: String + """Path to blobs - required for filesystem blob storage""" + blobsPath: String + """Where to store blobs""" + blobsStorage: BlobsStorageType """Whether to calculate MD5 checksums for scene video files""" calculateMD5: Boolean """Hash algorithm to use for generated file naming""" @@ -154,6 +167,10 @@ type ConfigGeneralResult { scrapersPath: String! """Path to cache""" cachePath: String! + """Path to blobs - required for filesystem blob storage""" + blobsPath: String! + """Where to store blobs""" + blobsStorage: BlobsStorageType! """Whether to calculate MD5 checksums for scene video files""" calculateMD5: Boolean! """Hash algorithm to use for generated file naming""" diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index bf3ee2566..ecde11eac 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -1,6 +1,7 @@ scalar Upload input GenerateMetadataInput { + covers: Boolean sprites: Boolean previews: Boolean imagePreviews: Boolean @@ -37,6 +38,7 @@ input GeneratePreviewOptionsInput { } type GenerateMetadataOptions { + covers: Boolean sprites: Boolean previews: Boolean imagePreviews: Boolean @@ -84,6 +86,8 @@ input ScanMetadataInput { """Strip file extension from title""" stripFileExtension: Boolean @deprecated(reason: "Not implemented") + """Generate covers during scan""" + scanGenerateCovers: Boolean """Generate previews during scan""" scanGeneratePreviews: Boolean """Generate image previews during scan""" @@ -101,9 +105,11 @@ input ScanMetadataInput { type ScanMetadataOptions { """Set name, date, details from metadata (if present)""" - useFileMetadata: Boolean! + useFileMetadata: Boolean! @deprecated(reason: "Not implemented") """Strip file extension from title""" - stripFileExtension: Boolean! + stripFileExtension: Boolean! @deprecated(reason: "Not implemented") + """Generate covers during scan""" + scanGenerateCovers: Boolean! """Generate previews during scan""" scanGeneratePreviews: Boolean! """Generate image previews during scan""" diff --git a/graphql/schema/types/migration.graphql b/graphql/schema/types/migration.graphql new file mode 100644 index 000000000..231f79555 --- /dev/null +++ b/graphql/schema/types/migration.graphql @@ -0,0 +1,11 @@ +input MigrateSceneScreenshotsInput { + # if true, delete screenshot files after migrating + deleteFiles: Boolean + # if true, overwrite existing covers with the covers from the screenshots directory + overwriteExisting: Boolean +} + +input MigrateBlobsInput { + # if true, delete blob data from old storage system + deleteOld: Boolean +} diff --git a/internal/api/resolver_mutation_configure.go b/internal/api/resolver_mutation_configure.go index 81c3138f5..5342182f9 100644 --- a/internal/api/resolver_mutation_configure.go +++ b/internal/api/resolver_mutation_configure.go @@ -134,6 +134,28 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen refreshStreamManager = true } + refreshBlobStorage := false + existingBlobsPath := c.GetBlobsPath() + if input.BlobsPath != nil && existingBlobsPath != *input.BlobsPath { + if err := validateDir(config.BlobsPath, *input.BlobsPath, true); err != nil { + return makeConfigGeneralResult(), err + } + + c.Set(config.BlobsPath, input.BlobsPath) + refreshBlobStorage = true + } + + if input.BlobsStorage != nil && *input.BlobsStorage != c.GetBlobsStorage() { + if *input.BlobsStorage == config.BlobStorageTypeFilesystem && c.GetBlobsPath() == "" { + return makeConfigGeneralResult(), fmt.Errorf("blobs path must be set when using filesystem storage") + } + + // TODO - migrate between systems + c.Set(config.BlobsStorage, input.BlobsStorage) + + refreshBlobStorage = true + } + if input.VideoFileNamingAlgorithm != nil && *input.VideoFileNamingAlgorithm != c.GetVideoFileNamingAlgorithm() { calculateMD5 := c.IsCalculateMD5() if input.CalculateMd5 != nil { @@ -336,6 +358,9 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input ConfigGen if refreshStreamManager { manager.GetInstance().RefreshStreamManager() } + if refreshBlobStorage { + manager.GetInstance().SetBlobStoreOptions() + } return makeConfigGeneralResult(), nil } @@ -530,7 +555,7 @@ func (r *mutationResolver) ConfigureDefaults(ctx context.Context, input ConfigDe } if input.Scan != nil { - c.Set(config.DefaultScanSettings, input.Scan.ScanMetadataOptions) + c.Set(config.DefaultScanSettings, input.Scan) } if input.AutoTag != nil { diff --git a/internal/api/resolver_mutation_migrate.go b/internal/api/resolver_mutation_migrate.go new file mode 100644 index 000000000..477f46b44 --- /dev/null +++ b/internal/api/resolver_mutation_migrate.go @@ -0,0 +1,40 @@ +package api + +import ( + "context" + "strconv" + + "github.com/stashapp/stash/internal/manager" + "github.com/stashapp/stash/internal/manager/task" + "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/utils" +) + +func (r *mutationResolver) MigrateSceneScreenshots(ctx context.Context, input MigrateSceneScreenshotsInput) (string, error) { + db := manager.GetInstance().Database + t := &task.MigrateSceneScreenshotsJob{ + ScreenshotsPath: manager.GetInstance().Paths.Generated.Screenshots, + Input: scene.MigrateSceneScreenshotsInput{ + DeleteFiles: utils.IsTrue(input.DeleteFiles), + OverwriteExisting: utils.IsTrue(input.OverwriteExisting), + }, + SceneRepo: db.Scene, + TxnManager: db, + } + jobID := manager.GetInstance().JobManager.Add(ctx, "Migrating scene screenshots to blobs...", t) + + return strconv.Itoa(jobID), nil +} + +func (r *mutationResolver) MigrateBlobs(ctx context.Context, input MigrateBlobsInput) (string, error) { + db := manager.GetInstance().Database + t := &task.MigrateBlobsJob{ + TxnManager: db, + BlobStore: db.Blobs, + Vacuumer: db, + DeleteOld: utils.IsTrue(input.DeleteOld), + } + jobID := manager.GetInstance().JobManager.Add(ctx, "Migrating blobs...", t) + + return strconv.Itoa(jobID), nil +} diff --git a/internal/api/resolver_mutation_movie.go b/internal/api/resolver_mutation_movie.go index f3d3e529d..009e9bc92 100644 --- a/internal/api/resolver_mutation_movie.go +++ b/internal/api/resolver_mutation_movie.go @@ -111,7 +111,13 @@ func (r *mutationResolver) MovieCreate(ctx context.Context, input MovieCreateInp // update image table if len(frontimageData) > 0 { - if err := qb.UpdateImages(ctx, movie.ID, frontimageData, backimageData); err != nil { + if err := qb.UpdateFrontImage(ctx, movie.ID, frontimageData); err != nil { + return err + } + } + + if len(backimageData) > 0 { + if err := qb.UpdateBackImage(ctx, movie.ID, backimageData); err != nil { return err } } @@ -184,35 +190,15 @@ func (r *mutationResolver) MovieUpdate(ctx context.Context, input MovieUpdateInp } // update image table - if frontImageIncluded || backImageIncluded { - if !frontImageIncluded { - frontimageData, err = qb.GetFrontImage(ctx, updatedMovie.ID) - if err != nil { - return err - } - } - if !backImageIncluded { - backimageData, err = qb.GetBackImage(ctx, updatedMovie.ID) - if err != nil { - return err - } + if frontImageIncluded { + if err := qb.UpdateFrontImage(ctx, movie.ID, frontimageData); err != nil { + return err } + } - if len(frontimageData) == 0 && len(backimageData) == 0 { - // both images are being nulled. Destroy them. - if err := qb.DestroyImages(ctx, movie.ID); err != nil { - return err - } - } else { - // HACK - if front image is null and back image is not null, then set the front image - // to the default image since we can't have a null front image and a non-null back image - if frontimageData == nil && backimageData != nil { - frontimageData, _ = utils.ProcessImageInput(ctx, models.DefaultMovieImage) - } - - if err := qb.UpdateImages(ctx, movie.ID, frontimageData, backimageData); err != nil { - return err - } + if backImageIncluded { + if err := qb.UpdateBackImage(ctx, movie.ID, backimageData); err != nil { + return err } } diff --git a/internal/api/resolver_mutation_scene.go b/internal/api/resolver_mutation_scene.go index 3abc917a9..dfdb29507 100644 --- a/internal/api/resolver_mutation_scene.go +++ b/internal/api/resolver_mutation_scene.go @@ -9,14 +9,12 @@ import ( "time" "github.com/stashapp/stash/internal/manager" - "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/sliceutil/intslice" "github.com/stashapp/stash/pkg/sliceutil/stringslice" - "github.com/stashapp/stash/pkg/txn" "github.com/stashapp/stash/pkg/utils" ) @@ -320,13 +318,6 @@ func (r *mutationResolver) sceneUpdateCoverImage(ctx context.Context, s *models. if err := qb.UpdateCover(ctx, s.ID, coverImageData); err != nil { return err } - - if s.Path != "" { - // update the file-based screenshot after commit - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { - return scene.SetScreenshot(manager.GetInstance().Paths, s.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm()), coverImageData) - }) - } } return nil diff --git a/internal/api/resolver_mutation_stash_box.go b/internal/api/resolver_mutation_stash_box.go index c01625688..9abb7179c 100644 --- a/internal/api/resolver_mutation_stash_box.go +++ b/internal/api/resolver_mutation_stash_box.go @@ -62,7 +62,7 @@ func (r *mutationResolver) SubmitStashBoxSceneDraft(ctx context.Context, input S return fmt.Errorf("scene with id %d not found", id) } - cover, err := r.sceneService.GetCover(ctx, scene) + cover, err := qb.GetCover(ctx, id) if err != nil { return fmt.Errorf("getting scene cover: %w", err) } diff --git a/internal/api/resolver_mutation_studio.go b/internal/api/resolver_mutation_studio.go index 98c871323..f9862d9be 100644 --- a/internal/api/resolver_mutation_studio.go +++ b/internal/api/resolver_mutation_studio.go @@ -176,15 +176,10 @@ func (r *mutationResolver) StudioUpdate(ctx context.Context, input StudioUpdateI } // update image table - if len(imageData) > 0 { + if imageIncluded { if err := qb.UpdateImage(ctx, s.ID, imageData); err != nil { return err } - } else if imageIncluded { - // must be unsetting - if err := qb.DestroyImage(ctx, s.ID); err != nil { - return err - } } // Save the stash_ids diff --git a/internal/api/resolver_mutation_tag.go b/internal/api/resolver_mutation_tag.go index 47986c56e..04f10ce88 100644 --- a/internal/api/resolver_mutation_tag.go +++ b/internal/api/resolver_mutation_tag.go @@ -208,15 +208,10 @@ func (r *mutationResolver) TagUpdate(ctx context.Context, input TagUpdateInput) } // update image table - if len(imageData) > 0 { + if imageIncluded { if err := qb.UpdateImage(ctx, tagID, imageData); err != nil { return err } - } else if imageIncluded { - // must be unsetting - if err := qb.DestroyImage(ctx, tagID); err != nil { - return err - } } if translator.hasField("aliases") { diff --git a/internal/api/resolver_query_configuration.go b/internal/api/resolver_query_configuration.go index 65b922d3e..fd598ce92 100644 --- a/internal/api/resolver_query_configuration.go +++ b/internal/api/resolver_query_configuration.go @@ -91,6 +91,8 @@ func makeConfigGeneralResult() *ConfigGeneralResult { ConfigFilePath: config.GetConfigFile(), ScrapersPath: config.GetScrapersPath(), CachePath: config.GetCachePath(), + BlobsPath: config.GetBlobsPath(), + BlobsStorage: config.GetBlobsStorage(), CalculateMd5: config.IsCalculateMD5(), VideoFileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), ParallelTasks: config.GetParallelTasks(), diff --git a/internal/api/routes_movie.go b/internal/api/routes_movie.go index c29718566..7b77586a6 100644 --- a/internal/api/routes_movie.go +++ b/internal/api/routes_movie.go @@ -42,8 +42,9 @@ func (rs movieRoutes) FrontImage(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { readTxnErr := txn.WithReadTxn(r.Context(), rs.txnManager, func(ctx context.Context) error { - image, _ = rs.movieFinder.GetFrontImage(ctx, movie.ID) - return nil + var err error + image, err = rs.movieFinder.GetFrontImage(ctx, movie.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return @@ -68,8 +69,9 @@ func (rs movieRoutes) BackImage(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { readTxnErr := txn.WithReadTxn(r.Context(), rs.txnManager, func(ctx context.Context) error { - image, _ = rs.movieFinder.GetBackImage(ctx, movie.ID) - return nil + var err error + image, err = rs.movieFinder.GetBackImage(ctx, movie.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return diff --git a/internal/api/routes_performer.go b/internal/api/routes_performer.go index c8295467a..1717e99f9 100644 --- a/internal/api/routes_performer.go +++ b/internal/api/routes_performer.go @@ -42,8 +42,9 @@ func (rs performerRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { readTxnErr := txn.WithReadTxn(r.Context(), rs.txnManager, func(ctx context.Context) error { - image, _ = rs.performerFinder.GetImage(ctx, performer.ID) - return nil + var err error + image, err = rs.performerFinder.GetImage(ctx, performer.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return diff --git a/internal/api/routes_studio.go b/internal/api/routes_studio.go index 2ddeb51a3..a77763e8d 100644 --- a/internal/api/routes_studio.go +++ b/internal/api/routes_studio.go @@ -42,8 +42,9 @@ func (rs studioRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { readTxnErr := txn.WithReadTxn(r.Context(), rs.txnManager, func(ctx context.Context) error { - image, _ = rs.studioFinder.GetImage(ctx, studio.ID) - return nil + var err error + image, err = rs.studioFinder.GetImage(ctx, studio.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return diff --git a/internal/api/routes_tag.go b/internal/api/routes_tag.go index 1f72928c2..e3ee439e9 100644 --- a/internal/api/routes_tag.go +++ b/internal/api/routes_tag.go @@ -42,8 +42,9 @@ func (rs tagRoutes) Image(w http.ResponseWriter, r *http.Request) { var image []byte if defaultParam != "true" { readTxnErr := txn.WithReadTxn(r.Context(), rs.txnManager, func(ctx context.Context) error { - image, _ = rs.tagFinder.GetImage(ctx, tag.ID) - return nil + var err error + image, err = rs.tagFinder.GetImage(ctx, tag.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return diff --git a/internal/identify/identify.go b/internal/identify/identify.go index c828f4164..04eccb7b0 100644 --- a/internal/identify/identify.go +++ b/internal/identify/identify.go @@ -35,7 +35,6 @@ type SceneIdentifier struct { DefaultOptions *MetadataOptions Sources []ScraperSource - ScreenshotSetter scene.ScreenshotSetter SceneUpdatePostHookExecutor SceneUpdatePostHookExecutor } @@ -216,7 +215,7 @@ func (t *SceneIdentifier) modifyScene(ctx context.Context, txnManager txn.Manage return nil } - if _, err := updater.Update(ctx, t.SceneReaderUpdater, t.ScreenshotSetter); err != nil { + if _, err := updater.Update(ctx, t.SceneReaderUpdater); err != nil { return fmt.Errorf("error updating scene: %w", err) } diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index 1ba0ce5b3..d5c6b7ac6 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -31,12 +31,15 @@ const ( BackupDirectoryPath = "backup_directory_path" Generated = "generated" Metadata = "metadata" + BlobsPath = "blobs_path" Downloads = "downloads" ApiKey = "api_key" Username = "username" Password = "password" MaxSessionAge = "max_session_age" + BlobsStorage = "blobs_storage" + DefaultMaxSessionAge = 60 * 60 * 1 // 1 hours Database = "database" @@ -551,6 +554,22 @@ func (i *Instance) GetGeneratedPath() string { return i.getString(Generated) } +func (i *Instance) GetBlobsPath() string { + return i.getString(BlobsPath) +} + +func (i *Instance) GetBlobsStorage() BlobsStorageType { + ret := BlobsStorageType(i.getString(BlobsStorage)) + + if !ret.IsValid() { + // default to database storage + // for legacy systems this is probably the safer option + ret = BlobStorageTypeDatabase + } + + return ret +} + func (i *Instance) GetMetadataPath() string { return i.getString(Metadata) } @@ -1458,6 +1477,12 @@ func (i *Instance) Validate() error { } } + if i.GetBlobsStorage() == BlobStorageTypeFilesystem && i.viper(BlobsPath).GetString(BlobsPath) == "" { + return MissingConfigError{ + missingFields: []string{BlobsPath}, + } + } + return nil } diff --git a/internal/manager/config/enums.go b/internal/manager/config/enums.go new file mode 100644 index 000000000..c1e045344 --- /dev/null +++ b/internal/manager/config/enums.go @@ -0,0 +1,50 @@ +package config + +import ( + "fmt" + "io" + "strconv" +) + +type BlobsStorageType string + +const ( + // Database + BlobStorageTypeDatabase BlobsStorageType = "DATABASE" + // Filesystem + BlobStorageTypeFilesystem BlobsStorageType = "FILESYSTEM" +) + +var AllBlobStorageType = []BlobsStorageType{ + BlobStorageTypeDatabase, + BlobStorageTypeFilesystem, +} + +func (e BlobsStorageType) IsValid() bool { + switch e { + case BlobStorageTypeDatabase, BlobStorageTypeFilesystem: + return true + } + return false +} + +func (e BlobsStorageType) String() string { + return string(e) +} + +func (e *BlobsStorageType) UnmarshalGQL(v interface{}) error { + str, ok := v.(string) + if !ok { + return fmt.Errorf("enums must be strings") + } + + *e = BlobsStorageType(str) + if !e.IsValid() { + return fmt.Errorf("%s is not a valid BlobStorageType", str) + } + return nil +} + +func (e BlobsStorageType) MarshalGQL(w io.Writer) { + fmt.Fprint(w, strconv.Quote(e.String())) +} diff --git a/internal/manager/config/tasks.go b/internal/manager/config/tasks.go index 2f69c8a50..1e541fcc5 100644 --- a/internal/manager/config/tasks.go +++ b/internal/manager/config/tasks.go @@ -7,6 +7,8 @@ type ScanMetadataOptions struct { // Strip file extension from title // Deprecated: not implemented StripFileExtension bool `json:"stripFileExtension"` + // Generate scene covers during scan + ScanGenerateCovers bool `json:"scanGenerateCovers"` // Generate previews during scan ScanGeneratePreviews bool `json:"scanGeneratePreviews"` // Generate image previews during scan diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 1ed4d163a..a591e98ab 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -26,11 +26,9 @@ import ( "github.com/stashapp/stash/pkg/image" "github.com/stashapp/stash/pkg/job" "github.com/stashapp/stash/pkg/logger" - "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" - "github.com/stashapp/stash/pkg/scene/generate" "github.com/stashapp/stash/pkg/scraper" "github.com/stashapp/stash/pkg/session" "github.com/stashapp/stash/pkg/sqlite" @@ -102,6 +100,8 @@ type SetupInput struct { GeneratedLocation string `json:"generatedLocation"` // Empty to indicate default CacheLocation string `json:"cacheLocation"` + // Empty to indicate database storage for blobs + BlobsLocation string `json:"blobsLocation"` } type Manager struct { @@ -290,20 +290,6 @@ func galleryFileFilter(ctx context.Context, f file.File) bool { return isZip(f.Base().Basename) } -type coverGenerator struct { -} - -func (g *coverGenerator) GenerateCover(ctx context.Context, scene *models.Scene, f *file.VideoFile) error { - gg := generate.Generator{ - Encoder: instance.FFMPEG, - FFMpegConfig: instance.Config, - LockManager: instance.ReadLockManager, - ScenePaths: instance.Paths.Scene, - } - - return gg.Screenshot(ctx, f.Path, scene.GetHash(instance.Config.GetVideoFileNamingAlgorithm()), f.Width, f.Duration, generate.ScreenshotOptions{}) -} - func makeScanner(db *sqlite.Database, pluginCache *plugin.Cache) *file.Scanner { return &file.Scanner{ Repository: file.Repository{ @@ -458,7 +444,7 @@ func (s *Manager) PostInit(ctx context.Context) error { logger.Warnf("could not set initial configuration: %v", err) } - *s.Paths = paths.NewPaths(s.Config.GetGeneratedPath()) + *s.Paths = paths.NewPaths(s.Config.GetGeneratedPath(), s.Config.GetBlobsPath()) s.RefreshConfig() s.SessionStore = session.NewStore(s.Config) s.PluginCache.RegisterSessionStore(s.SessionStore) @@ -467,6 +453,8 @@ func (s *Manager) PostInit(ctx context.Context) error { logger.Errorf("Error reading plugin configs: %s", err.Error()) } + s.SetBlobStoreOptions() + s.ScraperCache = instance.initScraperCache() writeStashIcon() @@ -509,6 +497,17 @@ func (s *Manager) PostInit(ctx context.Context) error { return nil } +func (s *Manager) SetBlobStoreOptions() { + storageType := s.Config.GetBlobsStorage() + blobsPath := s.Config.GetBlobsPath() + + s.Database.SetBlobStoreOptions(sqlite.BlobStoreOptions{ + UseFilesystem: storageType == config.BlobStorageTypeFilesystem, + UseDatabase: storageType == config.BlobStorageTypeDatabase, + Path: blobsPath, + }) +} + func writeStashIcon() { p := FaviconProvider{ UIBox: ui.UIBox, @@ -540,7 +539,7 @@ func (s *Manager) initScraperCache() *scraper.Cache { } func (s *Manager) RefreshConfig() { - *s.Paths = paths.NewPaths(s.Config.GetGeneratedPath()) + *s.Paths = paths.NewPaths(s.Config.GetGeneratedPath(), s.Config.GetBlobsPath()) config := s.Config if config.Validate() == nil { if err := fsutil.EnsureDir(s.Paths.Generated.Screenshots); err != nil { @@ -617,7 +616,7 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error { configDir := filepath.Dir(configFile) if exists, _ := fsutil.DirExists(configDir); !exists { - if err := os.Mkdir(configDir, 0755); err != nil { + if err := os.MkdirAll(configDir, 0755); err != nil { return fmt.Errorf("error creating config directory: %v", err) } } @@ -632,7 +631,7 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error { // create the generated directory if it does not exist if !c.HasOverride(config.Generated) { if exists, _ := fsutil.DirExists(input.GeneratedLocation); !exists { - if err := os.Mkdir(input.GeneratedLocation, 0755); err != nil { + if err := os.MkdirAll(input.GeneratedLocation, 0755); err != nil { return fmt.Errorf("error creating generated directory: %v", err) } } @@ -643,7 +642,7 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error { // create the cache directory if it does not exist if !c.HasOverride(config.Cache) { if exists, _ := fsutil.DirExists(input.CacheLocation); !exists { - if err := os.Mkdir(input.CacheLocation, 0755); err != nil { + if err := os.MkdirAll(input.CacheLocation, 0755); err != nil { return fmt.Errorf("error creating cache directory: %v", err) } } @@ -651,6 +650,22 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error { s.Config.Set(config.Cache, input.CacheLocation) } + // if blobs path was provided then use filesystem based blob storage + if input.BlobsLocation != "" { + if !c.HasOverride(config.BlobsPath) { + if exists, _ := fsutil.DirExists(input.BlobsLocation); !exists { + if err := os.MkdirAll(input.BlobsLocation, 0755); err != nil { + return fmt.Errorf("error creating blobs directory: %v", err) + } + } + } + + s.Config.Set(config.BlobsPath, input.BlobsLocation) + s.Config.Set(config.BlobsStorage, config.BlobStorageTypeFilesystem) + } else { + s.Config.Set(config.BlobsStorage, config.BlobStorageTypeDatabase) + } + // set the configuration if !c.HasOverride(config.Database) { s.Config.Set(config.Database, input.DatabaseFile) diff --git a/internal/manager/manager_tasks.go b/internal/manager/manager_tasks.go index 33354073d..de4807760 100644 --- a/internal/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -194,11 +194,11 @@ func (s *Manager) generateScreenshot(ctx context.Context, sceneId string, at *fl return } - task := GenerateScreenshotTask{ - txnManager: s.Repository, - Scene: *scene, - ScreenshotAt: at, - fileNamingAlgorithm: config.GetInstance().GetVideoFileNamingAlgorithm(), + task := GenerateCoverTask{ + txnManager: s.Repository, + Scene: *scene, + ScreenshotAt: at, + Overwrite: true, } task.Start(ctx) diff --git a/internal/manager/repository.go b/internal/manager/repository.go index 59428b146..c6ea17f85 100644 --- a/internal/manager/repository.go +++ b/internal/manager/repository.go @@ -102,8 +102,6 @@ type SceneService interface { AssignFile(ctx context.Context, sceneID int, fileID file.ID) error Merge(ctx context.Context, sourceIDs []int, destinationID int, values models.ScenePartial) error Destroy(ctx context.Context, scene *models.Scene, fileDeleter *scene.FileDeleter, deleteGenerated, deleteFile bool) error - - GetCover(ctx context.Context, scene *models.Scene) ([]byte, error) } type ImageService interface { diff --git a/internal/manager/running_streams.go b/internal/manager/running_streams.go index 83f93c2ea..5c95743b7 100644 --- a/internal/manager/running_streams.go +++ b/internal/manager/running_streams.go @@ -54,32 +54,32 @@ func (s *SceneServer) StreamSceneDirect(scene *models.Scene, w http.ResponseWrit func (s *SceneServer) ServeScreenshot(scene *models.Scene, w http.ResponseWriter, r *http.Request) { const defaultSceneImage = "scene/scene.svg" - if scene.Path != "" { - filepath := GetInstance().Paths.Scene.GetScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm())) - - // fall back to the scene image blob if the file isn't present - screenshotExists, _ := fsutil.FileExists(filepath) - if screenshotExists { - http.ServeFile(w, r, filepath) - return - } - } - var cover []byte readTxnErr := txn.WithReadTxn(r.Context(), s.TxnManager, func(ctx context.Context) error { - cover, _ = s.SceneCoverGetter.GetCover(ctx, scene.ID) - return nil + var err error + cover, err = s.SceneCoverGetter.GetCover(ctx, scene.ID) + return err }) if errors.Is(readTxnErr, context.Canceled) { return } if readTxnErr != nil { logger.Warnf("read transaction error on fetch screenshot: %v", readTxnErr) - http.Error(w, readTxnErr.Error(), http.StatusInternalServerError) - return } if cover == nil { + // fallback to legacy image if present + if scene.Path != "" { + filepath := GetInstance().Paths.Scene.GetLegacyScreenshotPath(scene.GetHash(config.GetInstance().GetVideoFileNamingAlgorithm())) + + // fall back to the scene image blob if the file isn't present + screenshotExists, _ := fsutil.FileExists(filepath) + if screenshotExists { + http.ServeFile(w, r, filepath) + return + } + } + // fallback to default cover if none found // should always be there f, _ := static.Scene.Open(defaultSceneImage) diff --git a/internal/manager/task/migrate_blobs.go b/internal/manager/task/migrate_blobs.go new file mode 100644 index 000000000..4cda65725 --- /dev/null +++ b/internal/manager/task/migrate_blobs.go @@ -0,0 +1,129 @@ +package task + +import ( + "context" + "fmt" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/txn" +) + +type BlobStoreMigrator interface { + Count(ctx context.Context) (int, error) + FindBlobs(ctx context.Context, n uint, lastChecksum string) ([]string, error) + MigrateBlob(ctx context.Context, checksum string, deleteOld bool) error +} + +type Vacuumer interface { + Vacuum(ctx context.Context) error +} + +type MigrateBlobsJob struct { + TxnManager txn.Manager + BlobStore BlobStoreMigrator + Vacuumer Vacuumer + DeleteOld bool +} + +func (j *MigrateBlobsJob) Execute(ctx context.Context, progress *job.Progress) { + var ( + count int + err error + ) + progress.ExecuteTask("Counting blobs", func() { + count, err = j.countBlobs(ctx) + progress.SetTotal(count) + }) + + if err != nil { + logger.Errorf("Error counting blobs: %s", err.Error()) + return + } + + if count == 0 { + logger.Infof("No blobs to migrate") + return + } + + logger.Infof("Migrating %d blobs", count) + + progress.ExecuteTask(fmt.Sprintf("Migrating %d blobs", count), func() { + err = j.migrateBlobs(ctx, progress) + }) + + if job.IsCancelled(ctx) { + logger.Info("Cancelled migrating blobs") + return + } + + if err != nil { + logger.Errorf("Error migrating blobs: %v", err) + return + } + + // run a vacuum to reclaim space + progress.ExecuteTask("Vacuuming database", func() { + err = j.Vacuumer.Vacuum(ctx) + if err != nil { + logger.Errorf("Error vacuuming database: %v", err) + } + }) + + logger.Infof("Finished migrating blobs") +} + +func (j *MigrateBlobsJob) countBlobs(ctx context.Context) (int, error) { + var count int + if err := txn.WithReadTxn(ctx, j.TxnManager, func(ctx context.Context) error { + var err error + count, err = j.BlobStore.Count(ctx) + return err + }); err != nil { + return 0, err + } + + return count, nil +} + +func (j *MigrateBlobsJob) migrateBlobs(ctx context.Context, progress *job.Progress) error { + lastChecksum := "" + batch, err := j.getBatch(ctx, lastChecksum) + + for len(batch) > 0 && err == nil && ctx.Err() == nil { + for _, checksum := range batch { + if ctx.Err() != nil { + return nil + } + + lastChecksum = checksum + + progress.ExecuteTask("Migrating blob "+checksum, func() { + defer progress.Increment() + + if err := txn.WithTxn(ctx, j.TxnManager, func(ctx context.Context) error { + return j.BlobStore.MigrateBlob(ctx, checksum, j.DeleteOld) + }); err != nil { + logger.Errorf("Error migrating blob %s: %v", checksum, err) + } + }) + } + + batch, err = j.getBatch(ctx, lastChecksum) + } + + return err +} + +func (j *MigrateBlobsJob) getBatch(ctx context.Context, lastChecksum string) ([]string, error) { + const batchSize = 1000 + + var batch []string + err := txn.WithReadTxn(ctx, j.TxnManager, func(ctx context.Context) error { + var err error + batch, err = j.BlobStore.FindBlobs(ctx, batchSize, lastChecksum) + return err + }) + + return batch, err +} diff --git a/internal/manager/task/migrate_scene_screenshots.go b/internal/manager/task/migrate_scene_screenshots.go new file mode 100644 index 000000000..bd758391f --- /dev/null +++ b/internal/manager/task/migrate_scene_screenshots.go @@ -0,0 +1,135 @@ +package task + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/job" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/txn" +) + +type MigrateSceneScreenshotsJob struct { + ScreenshotsPath string + Input scene.MigrateSceneScreenshotsInput + SceneRepo scene.HashFinderCoverUpdater + TxnManager txn.Manager +} + +func (j *MigrateSceneScreenshotsJob) Execute(ctx context.Context, progress *job.Progress) { + var err error + progress.ExecuteTask("Counting files", func() { + var count int + count, err = j.countFiles(ctx) + progress.SetTotal(count) + }) + + if err != nil { + logger.Errorf("Error counting files: %s", err.Error()) + return + } + + progress.ExecuteTask("Migrating files", func() { + err = j.migrateFiles(ctx, progress) + }) + + if job.IsCancelled(ctx) { + logger.Info("Cancelled migrating scene screenshots") + return + } + + if err != nil { + logger.Errorf("Error migrating scene screenshots: %v", err) + return + } + + logger.Infof("Finished migrating scene screenshots") +} + +func (j *MigrateSceneScreenshotsJob) countFiles(ctx context.Context) (int, error) { + f, err := os.Open(j.ScreenshotsPath) + if err != nil { + return 0, err + } + defer f.Close() + + const batchSize = 1000 + ret := 0 + files, err := f.ReadDir(batchSize) + for err == nil && ctx.Err() == nil { + ret += len(files) + + files, err = f.ReadDir(batchSize) + } + + if errors.Is(err, io.EOF) { + // end of directory + return ret, nil + } + + return 0, err +} + +func (j *MigrateSceneScreenshotsJob) migrateFiles(ctx context.Context, progress *job.Progress) error { + f, err := os.Open(j.ScreenshotsPath) + if err != nil { + return err + } + defer f.Close() + + m := scene.ScreenshotMigrator{ + Options: j.Input, + SceneUpdater: j.SceneRepo, + TxnManager: j.TxnManager, + } + + const batchSize = 1000 + files, err := f.ReadDir(batchSize) + for err == nil && ctx.Err() == nil { + for _, f := range files { + if ctx.Err() != nil { + return nil + } + + progress.ExecuteTask("Migrating file "+f.Name(), func() { + defer progress.Increment() + + path := filepath.Join(j.ScreenshotsPath, f.Name()) + + // sanity check - only process files + if f.IsDir() { + logger.Warnf("Skipping directory %s", path) + return + } + + // ignore non-jpg files + if !strings.HasSuffix(f.Name(), ".jpg") { + return + } + + // ignore .thumb files + if strings.HasSuffix(f.Name(), ".thumb.jpg") { + return + } + + if err := m.MigrateScreenshots(ctx, path); err != nil { + logger.Errorf("Error migrating screenshots for %s: %v", path, err) + } + }) + } + + files, err = f.ReadDir(batchSize) + } + + if errors.Is(err, io.EOF) { + // end of directory + return nil + } + + return err +} diff --git a/internal/manager/task_generate.go b/internal/manager/task_generate.go index 70ae85a9c..c3b4f16f7 100644 --- a/internal/manager/task_generate.go +++ b/internal/manager/task_generate.go @@ -13,28 +13,28 @@ import ( "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene/generate" "github.com/stashapp/stash/pkg/sliceutil/stringslice" - "github.com/stashapp/stash/pkg/utils" ) type GenerateMetadataInput struct { - Sprites *bool `json:"sprites"` - Previews *bool `json:"previews"` - ImagePreviews *bool `json:"imagePreviews"` + Covers bool `json:"covers"` + Sprites bool `json:"sprites"` + Previews bool `json:"previews"` + ImagePreviews bool `json:"imagePreviews"` PreviewOptions *GeneratePreviewOptionsInput `json:"previewOptions"` - Markers *bool `json:"markers"` - MarkerImagePreviews *bool `json:"markerImagePreviews"` - MarkerScreenshots *bool `json:"markerScreenshots"` - Transcodes *bool `json:"transcodes"` + Markers bool `json:"markers"` + MarkerImagePreviews bool `json:"markerImagePreviews"` + MarkerScreenshots bool `json:"markerScreenshots"` + Transcodes bool `json:"transcodes"` // Generate transcodes even if not required - ForceTranscodes *bool `json:"forceTranscodes"` - Phashes *bool `json:"phashes"` - InteractiveHeatmapsSpeeds *bool `json:"interactiveHeatmapsSpeeds"` + ForceTranscodes bool `json:"forceTranscodes"` + Phashes bool `json:"phashes"` + InteractiveHeatmapsSpeeds bool `json:"interactiveHeatmapsSpeeds"` // scene ids to generate for SceneIDs []string `json:"sceneIDs"` // marker ids to generate for MarkerIDs []string `json:"markerIDs"` // overwrite existing media - Overwrite *bool `json:"overwrite"` + Overwrite bool `json:"overwrite"` } type GeneratePreviewOptionsInput struct { @@ -61,6 +61,7 @@ type GenerateJob struct { } type totalsGenerate struct { + covers int64 sprites int64 previews int64 imagePreviews int64 @@ -77,9 +78,7 @@ func (j *GenerateJob) Execute(ctx context.Context, progress *job.Progress) { var err error var markers []*models.SceneMarker - if j.input.Overwrite != nil { - j.overwrite = *j.input.Overwrite - } + j.overwrite = j.input.Overwrite j.fileNamingAlgo = config.GetInstance().GetVideoFileNamingAlgorithm() config := config.GetInstance() @@ -143,7 +142,7 @@ func (j *GenerateJob) Execute(ctx context.Context, progress *job.Progress) { return } - logger.Infof("Generating %d sprites %d previews %d image previews %d markers %d transcodes %d phashes %d heatmaps & speeds", totals.sprites, totals.previews, totals.imagePreviews, totals.markers, totals.transcodes, totals.phashes, totals.interactiveHeatmapSpeeds) + logger.Infof("Generating %d covers %d sprites %d previews %d image previews %d markers %d transcodes %d phashes %d heatmaps & speeds", totals.covers, totals.sprites, totals.previews, totals.imagePreviews, totals.markers, totals.transcodes, totals.phashes, totals.interactiveHeatmapSpeeds) progress.SetTotal(int(totals.tasks)) }() @@ -266,7 +265,20 @@ func getGeneratePreviewOptions(optionsInput GeneratePreviewOptionsInput) generat } func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, scene *models.Scene, queue chan<- Task, totals *totalsGenerate) { - if utils.IsTrue(j.input.Sprites) { + if j.input.Covers { + task := &GenerateCoverTask{ + txnManager: j.txnManager, + Scene: *scene, + } + + if j.overwrite || task.required(ctx) { + totals.covers++ + totals.tasks++ + queue <- task + } + } + + if j.input.Sprites { task := &GenerateSpriteTask{ Scene: *scene, Overwrite: j.overwrite, @@ -286,10 +298,10 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, } options := getGeneratePreviewOptions(*generatePreviewOptions) - if utils.IsTrue(j.input.Previews) { + if j.input.Previews { task := &GeneratePreviewTask{ Scene: *scene, - ImagePreview: utils.IsTrue(j.input.ImagePreviews), + ImagePreview: j.input.ImagePreviews, Options: options, Overwrite: j.overwrite, fileNamingAlgorithm: j.fileNamingAlgo, @@ -303,7 +315,7 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, addTask = true } - if utils.IsTrue(j.input.ImagePreviews) && (j.overwrite || !task.doesImagePreviewExist()) { + if j.input.ImagePreviews && (j.overwrite || !task.doesImagePreviewExist()) { totals.imagePreviews++ addTask = true } @@ -315,14 +327,14 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, } } - if utils.IsTrue(j.input.Markers) { + if j.input.Markers { task := &GenerateMarkersTask{ TxnManager: j.txnManager, Scene: scene, Overwrite: j.overwrite, fileNamingAlgorithm: j.fileNamingAlgo, - ImagePreview: utils.IsTrue(j.input.MarkerImagePreviews), - Screenshot: utils.IsTrue(j.input.MarkerScreenshots), + ImagePreview: j.input.MarkerImagePreviews, + Screenshot: j.input.MarkerScreenshots, generator: g, } @@ -336,8 +348,8 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, } } - if utils.IsTrue(j.input.Transcodes) { - forceTranscode := utils.IsTrue(j.input.ForceTranscodes) + if j.input.Transcodes { + forceTranscode := j.input.ForceTranscodes task := &GenerateTranscodeTask{ Scene: *scene, Overwrite: j.overwrite, @@ -352,7 +364,7 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, } } - if utils.IsTrue(j.input.Phashes) { + if j.input.Phashes { // generate for all files in scene for _, f := range scene.Files.List() { task := &GeneratePhashTask{ @@ -371,7 +383,7 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator, } } - if utils.IsTrue(j.input.InteractiveHeatmapsSpeeds) { + if j.input.InteractiveHeatmapsSpeeds { task := &GenerateInteractiveHeatmapSpeedTask{ Scene: *scene, Overwrite: j.overwrite, diff --git a/internal/manager/task_generate_screenshot.go b/internal/manager/task_generate_screenshot.go index 8b38e282a..b3ed51a78 100644 --- a/internal/manager/task_generate_screenshot.go +++ b/internal/manager/task_generate_screenshot.go @@ -3,25 +3,32 @@ package manager import ( "context" "fmt" - "io" - "os" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene/generate" ) -type GenerateScreenshotTask struct { - Scene models.Scene - ScreenshotAt *float64 - fileNamingAlgorithm models.HashAlgorithm - txnManager Repository +type GenerateCoverTask struct { + Scene models.Scene + ScreenshotAt *float64 + txnManager Repository + Overwrite bool } -func (t *GenerateScreenshotTask) Start(ctx context.Context) { +func (t *GenerateCoverTask) GetDescription() string { + return fmt.Sprintf("Generating cover for %s", t.Scene.GetTitle()) +} + +func (t *GenerateCoverTask) Start(ctx context.Context) { scenePath := t.Scene.Path + if err := t.txnManager.WithReadTxn(ctx, func(ctx context.Context) error { + return t.Scene.LoadPrimaryFile(ctx, t.txnManager.File) + }); err != nil { + logger.Error(err) + } + videoFile := t.Scene.Files.Primary() if videoFile == nil { return @@ -34,12 +41,8 @@ func (t *GenerateScreenshotTask) Start(ctx context.Context) { at = *t.ScreenshotAt } - checksum := t.Scene.GetHash(t.fileNamingAlgorithm) - normalPath := instance.Paths.Scene.GetScreenshotPath(checksum) - // we'll generate the screenshot, grab the generated data and set it - // in the database. We'll use SetSceneScreenshot to set the data - // which also generates the thumbnail + // in the database. logger.Debugf("Creating screenshot for %s", scenePath) @@ -51,35 +54,19 @@ func (t *GenerateScreenshotTask) Start(ctx context.Context) { Overwrite: true, } - if err := g.Screenshot(context.TODO(), videoFile.Path, checksum, videoFile.Width, videoFile.Duration, generate.ScreenshotOptions{ + coverImageData, err := g.Screenshot(context.TODO(), videoFile.Path, videoFile.Width, videoFile.Duration, generate.ScreenshotOptions{ At: &at, - }); err != nil { + }) + if err != nil { logger.Errorf("Error generating screenshot: %v", err) logErrorOutput(err) return } - f, err := os.Open(normalPath) - if err != nil { - logger.Errorf("Error reading screenshot: %s", err.Error()) - return - } - defer f.Close() - - coverImageData, err := io.ReadAll(f) - if err != nil { - logger.Errorf("Error reading screenshot: %s", err.Error()) - return - } - if err := t.txnManager.WithTxn(ctx, func(ctx context.Context) error { qb := t.txnManager.Scene updatedScene := models.NewScenePartial() - if err := scene.SetScreenshot(instance.Paths, checksum, coverImageData); err != nil { - return fmt.Errorf("error writing screenshot: %v", err) - } - // update the scene cover table if err := qb.UpdateCover(ctx, t.Scene.ID, coverImageData); err != nil { return fmt.Errorf("error setting screenshot: %v", err) @@ -96,3 +83,19 @@ func (t *GenerateScreenshotTask) Start(ctx context.Context) { logger.Error(err.Error()) } } + +// required returns true if the sprite needs to be generated +func (t GenerateCoverTask) required(ctx context.Context) bool { + if t.Overwrite { + return true + } + + // if the scene has a cover, then we don't need to generate it + hasCover, err := t.txnManager.Scene.HasCover(ctx, t.Scene.ID) + if err != nil { + logger.Errorf("Error getting cover: %v", err) + return false + } + + return !hasCover +} diff --git a/internal/manager/task_identify.go b/internal/manager/task_identify.go index 078e541ee..955dcb2b3 100644 --- a/internal/manager/task_identify.go +++ b/internal/manager/task_identify.go @@ -138,12 +138,8 @@ func (j *IdentifyJob) identifyScene(ctx context.Context, s *models.Scene, source PerformerCreator: instance.Repository.Performer, TagCreator: instance.Repository.Tag, - DefaultOptions: j.input.Options, - Sources: sources, - ScreenshotSetter: &scene.PathsCoverSetter{ - Paths: instance.Paths, - FileNamingAlgorithm: instance.Config.GetVideoFileNamingAlgorithm(), - }, + DefaultOptions: j.input.Options, + Sources: sources, SceneUpdatePostHookExecutor: j.postHookExecutor, } diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index 1d8419dcc..6b24af27b 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -194,22 +194,22 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool { } if isVideoFile { - // check if the screenshot file exists - hash := scene.GetHash(ff, f.videoFileNamingAlgorithm) - ssPath := instance.Paths.Scene.GetScreenshotPath(hash) - if exists, _ := fsutil.FileExists(ssPath); !exists { - // if not, check if the file is a primary file for a scene - scenes, err := f.SceneFinder.FindByPrimaryFileID(ctx, ff.Base().ID) - if err != nil { - // just ignore - return false - } + // TODO - check if the cover exists + // hash := scene.GetHash(ff, f.videoFileNamingAlgorithm) + // ssPath := instance.Paths.Scene.GetScreenshotPath(hash) + // if exists, _ := fsutil.FileExists(ssPath); !exists { + // // if not, check if the file is a primary file for a scene + // scenes, err := f.SceneFinder.FindByPrimaryFileID(ctx, ff.Base().ID) + // if err != nil { + // // just ignore + // return false + // } - if len(scenes) > 0 { - // if it is, then it needs to be re-generated - return true - } - } + // if len(scenes) > 0 { + // // if it is, then it needs to be re-generated + // return true + // } + // } // clean captions - scene handler handles this as well, but // unchanged files aren't processed by the scene handler @@ -349,7 +349,6 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre CreatorUpdater: db.Scene, PluginCache: pluginCache, CaptionUpdater: db.File, - CoverGenerator: &coverGenerator{}, ScanGenerator: &sceneGenerators{ input: options, taskQueue: taskQueue, @@ -485,5 +484,17 @@ func (g *sceneGenerators) Generate(ctx context.Context, s *models.Scene, f *file } } + if t.ScanGenerateCovers { + progress.AddTotal(1) + g.taskQueue.Add(fmt.Sprintf("Generating cover for %s", path), func(ctx context.Context) { + taskCover := GenerateCoverTask{ + Scene: *s, + txnManager: instance.Repository, + } + taskCover.Start(ctx) + progress.Increment() + }) + } + return nil } diff --git a/pkg/file/clean.go b/pkg/file/clean.go index 53c1c981e..44470c5a0 100644 --- a/pkg/file/clean.go +++ b/pkg/file/clean.go @@ -380,7 +380,7 @@ func (j *cleanJob) deleteFile(ctx context.Context, fileID ID, fn string) { // delete associated objects fileDeleter := NewDeleter() if err := txn.WithTxn(ctx, j.Repository, func(ctx context.Context) error { - fileDeleter.RegisterHooks(ctx, j.Repository) + fileDeleter.RegisterHooks(ctx) if err := j.fireHandlers(ctx, fileDeleter, fileID); err != nil { return err @@ -397,7 +397,7 @@ func (j *cleanJob) deleteFolder(ctx context.Context, folderID FolderID, fn strin // delete associated objects fileDeleter := NewDeleter() if err := txn.WithTxn(ctx, j.Repository, func(ctx context.Context) error { - fileDeleter.RegisterHooks(ctx, j.Repository) + fileDeleter.RegisterHooks(ctx) if err := j.fireFolderHandlers(ctx, fileDeleter, folderID); err != nil { return err diff --git a/pkg/file/delete.go b/pkg/file/delete.go index c71d73428..4c7a621b6 100644 --- a/pkg/file/delete.go +++ b/pkg/file/delete.go @@ -69,15 +69,13 @@ func NewDeleter() *Deleter { } // RegisterHooks registers post-commit and post-rollback hooks. -func (d *Deleter) RegisterHooks(ctx context.Context, mgr txn.Manager) { - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { +func (d *Deleter) RegisterHooks(ctx context.Context) { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { d.Commit() - return nil }) - txn.AddPostRollbackHook(ctx, func(ctx context.Context) error { + txn.AddPostRollbackHook(ctx, func(ctx context.Context) { d.Rollback() - return nil }) } diff --git a/pkg/file/fs.go b/pkg/file/fs.go index 0a24aaa53..09c7c7c8e 100644 --- a/pkg/file/fs.go +++ b/pkg/file/fs.go @@ -34,6 +34,26 @@ type FS interface { // OsFS is a file system backed by the OS. type OsFS struct{} +func (f *OsFS) Create(name string) (*os.File, error) { + return os.Create(name) +} + +func (f *OsFS) MkdirAll(path string, perm fs.FileMode) error { + return os.MkdirAll(path, perm) +} + +func (f *OsFS) Remove(name string) error { + return os.Remove(name) +} + +func (f *OsFS) Rename(oldpath, newpath string) error { + return os.Rename(oldpath, newpath) +} + +func (f *OsFS) RemoveAll(path string) error { + return os.RemoveAll(path) +} + func (f *OsFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) } diff --git a/pkg/file/scan.go b/pkg/file/scan.go index 18c1955d2..148f18691 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -508,12 +508,11 @@ func (s *scanJob) onNewFolder(ctx context.Context, file scanFile) (*Folder, erro } } - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { // log at the end so that if anything fails above due to a locked database // error and the transaction must be retried, then we shouldn't get multiple // logs of the same thing. logger.Infof("%s doesn't exist. Creating new folder entry...", file.Path) - return nil }) if err := s.Repository.FolderStore.Create(ctx, toCreate); err != nil { diff --git a/pkg/image/scan.go b/pkg/image/scan.go index c8d02c26f..4c5280f6b 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -143,15 +143,13 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File if h.ScanConfig.IsGenerateThumbnails() { // do this after the commit so that the transaction isn't held up - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { for _, s := range existing { if err := h.ThumbnailGenerator.GenerateThumbnail(ctx, s, imageFile); err != nil { // just log if cover generation fails. We can try again on rescan logger.Errorf("Error generating thumbnail for %s: %v", imageFile.Path, err) } } - - return nil }) } diff --git a/pkg/models/generate.go b/pkg/models/generate.go index 85685e078..2fc66248c 100644 --- a/pkg/models/generate.go +++ b/pkg/models/generate.go @@ -7,16 +7,17 @@ import ( ) type GenerateMetadataOptions struct { - Sprites *bool `json:"sprites"` - Previews *bool `json:"previews"` - ImagePreviews *bool `json:"imagePreviews"` + Covers bool `json:"covers"` + Sprites bool `json:"sprites"` + Previews bool `json:"previews"` + ImagePreviews bool `json:"imagePreviews"` PreviewOptions *GeneratePreviewOptions `json:"previewOptions"` - Markers *bool `json:"markers"` - MarkerImagePreviews *bool `json:"markerImagePreviews"` - MarkerScreenshots *bool `json:"markerScreenshots"` - Transcodes *bool `json:"transcodes"` - Phashes *bool `json:"phashes"` - InteractiveHeatmapsSpeeds *bool `json:"interactiveHeatmapsSpeeds"` + Markers bool `json:"markers"` + MarkerImagePreviews bool `json:"markerImagePreviews"` + MarkerScreenshots bool `json:"markerScreenshots"` + Transcodes bool `json:"transcodes"` + Phashes bool `json:"phashes"` + InteractiveHeatmapsSpeeds bool `json:"interactiveHeatmapsSpeeds"` } type GeneratePreviewOptions struct { diff --git a/pkg/models/mocks/MovieReaderWriter.go b/pkg/models/mocks/MovieReaderWriter.go index c125fc7b1..ad2171a66 100644 --- a/pkg/models/mocks/MovieReaderWriter.go +++ b/pkg/models/mocks/MovieReaderWriter.go @@ -137,20 +137,6 @@ func (_m *MovieReaderWriter) Destroy(ctx context.Context, id int) error { return r0 } -// DestroyImages provides a mock function with given fields: ctx, movieID -func (_m *MovieReaderWriter) DestroyImages(ctx context.Context, movieID int) error { - ret := _m.Called(ctx, movieID) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int) error); ok { - r0 = rf(ctx, movieID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Find provides a mock function with given fields: ctx, id func (_m *MovieReaderWriter) Find(ctx context.Context, id int) (*models.Movie, error) { ret := _m.Called(ctx, id) @@ -388,6 +374,34 @@ func (_m *MovieReaderWriter) Update(ctx context.Context, updatedMovie models.Mov return r0, r1 } +// UpdateBackImage provides a mock function with given fields: ctx, movieID, backImage +func (_m *MovieReaderWriter) UpdateBackImage(ctx context.Context, movieID int, backImage []byte) error { + ret := _m.Called(ctx, movieID, backImage) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int, []byte) error); ok { + r0 = rf(ctx, movieID, backImage) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UpdateFrontImage provides a mock function with given fields: ctx, movieID, frontImage +func (_m *MovieReaderWriter) UpdateFrontImage(ctx context.Context, movieID int, frontImage []byte) error { + ret := _m.Called(ctx, movieID, frontImage) + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context, int, []byte) error); ok { + r0 = rf(ctx, movieID, frontImage) + } else { + r0 = ret.Error(0) + } + + return r0 +} + // UpdateFull provides a mock function with given fields: ctx, updatedMovie func (_m *MovieReaderWriter) UpdateFull(ctx context.Context, updatedMovie models.Movie) (*models.Movie, error) { ret := _m.Called(ctx, updatedMovie) @@ -410,17 +424,3 @@ func (_m *MovieReaderWriter) UpdateFull(ctx context.Context, updatedMovie models return r0, r1 } - -// UpdateImages provides a mock function with given fields: ctx, movieID, frontImage, backImage -func (_m *MovieReaderWriter) UpdateImages(ctx context.Context, movieID int, frontImage []byte, backImage []byte) error { - ret := _m.Called(ctx, movieID, frontImage, backImage) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int, []byte, []byte) error); ok { - r0 = rf(ctx, movieID, frontImage, backImage) - } else { - r0 = ret.Error(0) - } - - return r0 -} diff --git a/pkg/models/mocks/SceneReaderWriter.go b/pkg/models/mocks/SceneReaderWriter.go index bb6e6d4a5..5f7191827 100644 --- a/pkg/models/mocks/SceneReaderWriter.go +++ b/pkg/models/mocks/SceneReaderWriter.go @@ -235,20 +235,6 @@ func (_m *SceneReaderWriter) Destroy(ctx context.Context, id int) error { return r0 } -// DestroyCover provides a mock function with given fields: ctx, sceneID -func (_m *SceneReaderWriter) DestroyCover(ctx context.Context, sceneID int) error { - ret := _m.Called(ctx, sceneID) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int) error); ok { - r0 = rf(ctx, sceneID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Duration provides a mock function with given fields: ctx func (_m *SceneReaderWriter) Duration(ctx context.Context) (float64, error) { ret := _m.Called(ctx) @@ -638,6 +624,27 @@ func (_m *SceneReaderWriter) GetTagIDs(ctx context.Context, relatedID int) ([]in return r0, r1 } +// HasCover provides a mock function with given fields: ctx, sceneID +func (_m *SceneReaderWriter) HasCover(ctx context.Context, sceneID int) (bool, error) { + ret := _m.Called(ctx, sceneID) + + var r0 bool + if rf, ok := ret.Get(0).(func(context.Context, int) bool); ok { + r0 = rf(ctx, sceneID) + } else { + r0 = ret.Get(0).(bool) + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, int) error); ok { + r1 = rf(ctx, sceneID) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // IncrementOCounter provides a mock function with given fields: ctx, id func (_m *SceneReaderWriter) IncrementOCounter(ctx context.Context, id int) (int, error) { ret := _m.Called(ctx, id) diff --git a/pkg/models/mocks/StudioReaderWriter.go b/pkg/models/mocks/StudioReaderWriter.go index 043bfdecc..8868efcc8 100644 --- a/pkg/models/mocks/StudioReaderWriter.go +++ b/pkg/models/mocks/StudioReaderWriter.go @@ -95,20 +95,6 @@ func (_m *StudioReaderWriter) Destroy(ctx context.Context, id int) error { return r0 } -// DestroyImage provides a mock function with given fields: ctx, studioID -func (_m *StudioReaderWriter) DestroyImage(ctx context.Context, studioID int) error { - ret := _m.Called(ctx, studioID) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int) error); ok { - r0 = rf(ctx, studioID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Find provides a mock function with given fields: ctx, id func (_m *StudioReaderWriter) Find(ctx context.Context, id int) (*models.Studio, error) { ret := _m.Called(ctx, id) diff --git a/pkg/models/mocks/TagReaderWriter.go b/pkg/models/mocks/TagReaderWriter.go index 1a53adf05..d67f4f00e 100644 --- a/pkg/models/mocks/TagReaderWriter.go +++ b/pkg/models/mocks/TagReaderWriter.go @@ -95,20 +95,6 @@ func (_m *TagReaderWriter) Destroy(ctx context.Context, id int) error { return r0 } -// DestroyImage provides a mock function with given fields: ctx, tagID -func (_m *TagReaderWriter) DestroyImage(ctx context.Context, tagID int) error { - ret := _m.Called(ctx, tagID) - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int) error); ok { - r0 = rf(ctx, tagID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - // Find provides a mock function with given fields: ctx, id func (_m *TagReaderWriter) Find(ctx context.Context, id int) (*models.Tag, error) { ret := _m.Called(ctx, id) diff --git a/pkg/models/model_movie.go b/pkg/models/model_movie.go index 756a6c936..00b87ad0f 100644 --- a/pkg/models/model_movie.go +++ b/pkg/models/model_movie.go @@ -22,6 +22,10 @@ type Movie struct { URL sql.NullString `db:"url" json:"url"` CreatedAt SQLiteTimestamp `db:"created_at" json:"created_at"` UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` + + // TODO - this is only here because of database code in the models package + FrontImageBlob sql.NullString `db:"front_image_blob" json:"-"` + BackImageBlob sql.NullString `db:"back_image_blob" json:"-"` } type MoviePartial struct { diff --git a/pkg/models/model_studio.go b/pkg/models/model_studio.go index 51a8d332c..989415293 100644 --- a/pkg/models/model_studio.go +++ b/pkg/models/model_studio.go @@ -19,6 +19,8 @@ type Studio struct { Rating sql.NullInt64 `db:"rating" json:"rating"` Details sql.NullString `db:"details" json:"details"` IgnoreAutoTag bool `db:"ignore_auto_tag" json:"ignore_auto_tag"` + // TODO - this is only here because of database code in the models package + ImageBlob sql.NullString `db:"image_blob" json:"-"` } type StudioPartial struct { diff --git a/pkg/models/model_tag.go b/pkg/models/model_tag.go index 043f84bd6..b12574155 100644 --- a/pkg/models/model_tag.go +++ b/pkg/models/model_tag.go @@ -6,12 +6,14 @@ import ( ) type Tag struct { - ID int `db:"id" json:"id"` - Name string `db:"name" json:"name"` // TODO make schema not null - Description sql.NullString `db:"description" json:"description"` - IgnoreAutoTag bool `db:"ignore_auto_tag" json:"ignore_auto_tag"` - CreatedAt SQLiteTimestamp `db:"created_at" json:"created_at"` - UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` + ID int `db:"id" json:"id"` + Name string `db:"name" json:"name"` // TODO make schema not null + Description sql.NullString `db:"description" json:"description"` + IgnoreAutoTag bool `db:"ignore_auto_tag" json:"ignore_auto_tag"` + // TODO - this is only here because of database code in the models package + ImageBlob sql.NullString `db:"image_blob" json:"-"` + CreatedAt SQLiteTimestamp `db:"created_at" json:"created_at"` + UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` } type TagPartial struct { diff --git a/pkg/models/movie.go b/pkg/models/movie.go index 8d58d70dd..adaa4fe03 100644 --- a/pkg/models/movie.go +++ b/pkg/models/movie.go @@ -50,8 +50,8 @@ type MovieWriter interface { Update(ctx context.Context, updatedMovie MoviePartial) (*Movie, error) UpdateFull(ctx context.Context, updatedMovie Movie) (*Movie, error) Destroy(ctx context.Context, id int) error - UpdateImages(ctx context.Context, movieID int, frontImage []byte, backImage []byte) error - DestroyImages(ctx context.Context, movieID int) error + UpdateFrontImage(ctx context.Context, movieID int, frontImage []byte) error + UpdateBackImage(ctx context.Context, movieID int, backImage []byte) error } type MovieReaderWriter interface { diff --git a/pkg/models/paths/paths.go b/pkg/models/paths/paths.go index 612d5e2b7..ed35bca56 100644 --- a/pkg/models/paths/paths.go +++ b/pkg/models/paths/paths.go @@ -11,14 +11,17 @@ type Paths struct { Scene *scenePaths SceneMarkers *sceneMarkerPaths + Blobs string } -func NewPaths(generatedPath string) Paths { +func NewPaths(generatedPath string, blobsPath string) Paths { p := Paths{} p.Generated = newGeneratedPaths(generatedPath) p.Scene = newScenePaths(p) p.SceneMarkers = newSceneMarkerPaths(p) + p.Blobs = blobsPath + return p } diff --git a/pkg/models/paths/paths_scenes.go b/pkg/models/paths/paths_scenes.go index d54fbfb59..003e63304 100644 --- a/pkg/models/paths/paths_scenes.go +++ b/pkg/models/paths/paths_scenes.go @@ -17,14 +17,10 @@ func newScenePaths(p Paths) *scenePaths { return &sp } -func (sp *scenePaths) GetScreenshotPath(checksum string) string { +func (sp *scenePaths) GetLegacyScreenshotPath(checksum string) string { return filepath.Join(sp.Screenshots, checksum+".jpg") } -func (sp *scenePaths) GetThumbnailScreenshotPath(checksum string) string { - return filepath.Join(sp.Screenshots, checksum+".thumb.jpg") -} - func (sp *scenePaths) GetTranscodePath(checksum string) string { return filepath.Join(sp.Transcodes, checksum+".mp4") } diff --git a/pkg/models/scene.go b/pkg/models/scene.go index 3d6842943..55a27606a 100644 --- a/pkg/models/scene.go +++ b/pkg/models/scene.go @@ -176,6 +176,7 @@ type SceneReader interface { All(ctx context.Context) ([]*Scene, error) Query(ctx context.Context, options SceneQueryOptions) (*SceneQueryResult, error) GetCover(ctx context.Context, sceneID int) ([]byte, error) + HasCover(ctx context.Context, sceneID int) (bool, error) } type SceneWriter interface { @@ -189,7 +190,6 @@ type SceneWriter interface { IncrementWatchCount(ctx context.Context, id int) (int, error) Destroy(ctx context.Context, id int) error UpdateCover(ctx context.Context, sceneID int, cover []byte) error - DestroyCover(ctx context.Context, sceneID int) error } type SceneReaderWriter interface { diff --git a/pkg/models/studio.go b/pkg/models/studio.go index 26443edf7..7ccf33be0 100644 --- a/pkg/models/studio.go +++ b/pkg/models/studio.go @@ -66,7 +66,6 @@ type StudioWriter interface { UpdateFull(ctx context.Context, updatedStudio Studio) (*Studio, error) Destroy(ctx context.Context, id int) error UpdateImage(ctx context.Context, studioID int, image []byte) error - DestroyImage(ctx context.Context, studioID int) error UpdateStashIDs(ctx context.Context, studioID int, stashIDs []StashID) error UpdateAliases(ctx context.Context, studioID int, aliases []string) error } diff --git a/pkg/models/tag.go b/pkg/models/tag.go index 440d147d3..601dfcc16 100644 --- a/pkg/models/tag.go +++ b/pkg/models/tag.go @@ -74,7 +74,6 @@ type TagWriter interface { UpdateFull(ctx context.Context, updatedTag Tag) (*Tag, error) Destroy(ctx context.Context, id int) error UpdateImage(ctx context.Context, tagID int, image []byte) error - DestroyImage(ctx context.Context, tagID int) error UpdateAliases(ctx context.Context, tagID int, aliases []string) error Merge(ctx context.Context, source []int, destination int) error UpdateParentTags(ctx context.Context, tagID int, parentIDs []int) error diff --git a/pkg/movie/import.go b/pkg/movie/import.go index 461df0f84..75bc28d4a 100644 --- a/pkg/movie/import.go +++ b/pkg/movie/import.go @@ -12,10 +12,15 @@ import ( "github.com/stashapp/stash/pkg/utils" ) +type ImageUpdater interface { + UpdateFrontImage(ctx context.Context, movieID int, frontImage []byte) error + UpdateBackImage(ctx context.Context, movieID int, backImage []byte) error +} + type NameFinderCreatorUpdater interface { NameFinderCreator UpdateFull(ctx context.Context, updatedMovie models.Movie) (*models.Movie, error) - UpdateImages(ctx context.Context, movieID int, frontImage []byte, backImage []byte) error + ImageUpdater } type Importer struct { @@ -126,8 +131,14 @@ func (i *Importer) createStudio(ctx context.Context, name string) (int, error) { func (i *Importer) PostImport(ctx context.Context, id int) error { if len(i.frontImageData) > 0 { - if err := i.ReaderWriter.UpdateImages(ctx, id, i.frontImageData, i.backImageData); err != nil { - return fmt.Errorf("error setting movie images: %v", err) + if err := i.ReaderWriter.UpdateFrontImage(ctx, id, i.frontImageData); err != nil { + return fmt.Errorf("error setting movie front image: %v", err) + } + } + + if len(i.backImageData) > 0 { + if err := i.ReaderWriter.UpdateBackImage(ctx, id, i.backImageData); err != nil { + return fmt.Errorf("error setting movie back image: %v", err) } } diff --git a/pkg/movie/import_test.go b/pkg/movie/import_test.go index 26b6d9f27..c33d4baa2 100644 --- a/pkg/movie/import_test.go +++ b/pkg/movie/import_test.go @@ -162,8 +162,9 @@ func TestImporterPostImport(t *testing.T) { updateMovieImageErr := errors.New("UpdateImages error") - readerWriter.On("UpdateImages", testCtx, movieID, frontImageBytes, backImageBytes).Return(nil).Once() - readerWriter.On("UpdateImages", testCtx, errImageID, frontImageBytes, backImageBytes).Return(updateMovieImageErr).Once() + readerWriter.On("UpdateFrontImage", testCtx, movieID, frontImageBytes).Return(nil).Once() + readerWriter.On("UpdateBackImage", testCtx, movieID, backImageBytes).Return(nil).Once() + readerWriter.On("UpdateFrontImage", testCtx, errImageID, frontImageBytes).Return(updateMovieImageErr).Once() err := i.PostImport(testCtx, movieID) assert.Nil(t, err) diff --git a/pkg/plugin/plugins.go b/pkg/plugin/plugins.go index 6cfb27bcd..8e9354d0b 100644 --- a/pkg/plugin/plugins.go +++ b/pkg/plugin/plugins.go @@ -210,9 +210,8 @@ func (c Cache) ExecutePostHooks(ctx context.Context, id int, hookType HookTrigge } func (c Cache) RegisterPostHooks(ctx context.Context, id int, hookType HookTriggerEnum, input interface{}, inputFields []string) { - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { c.ExecutePostHooks(ctx, id, hookType, input, inputFields) - return nil }) } diff --git a/pkg/scene/create.go b/pkg/scene/create.go index 83fd5e56c..c2345d2ef 100644 --- a/pkg/scene/create.go +++ b/pkg/scene/create.go @@ -7,10 +7,8 @@ import ( "time" "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" - "github.com/stashapp/stash/pkg/txn" ) func (s *Service) Create(ctx context.Context, input *models.Scene, fileIDs []file.ID, coverImage []byte) (*models.Scene, error) { @@ -55,18 +53,6 @@ func (s *Service) Create(ctx context.Context, input *models.Scene, fileIDs []fil if err := s.Repository.UpdateCover(ctx, ret.ID, coverImage); err != nil { return nil, fmt.Errorf("setting cover on new scene: %w", err) } - - // only update the cover image if provided and everything else was successful - // only do this if there is a file associated - if len(fileIDs) > 0 { - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { - if err := SetScreenshot(s.Paths, ret.GetHash(s.Config.GetVideoFileNamingAlgorithm()), coverImage); err != nil { - logger.Errorf("Error setting screenshot: %v", err) - } - - return nil - }) - } } s.PluginCache.RegisterPostHooks(ctx, ret.ID, plugin.SceneCreatePost, nil, nil) diff --git a/pkg/scene/delete.go b/pkg/scene/delete.go index 622b54377..f7fdda0c9 100644 --- a/pkg/scene/delete.go +++ b/pkg/scene/delete.go @@ -38,18 +38,6 @@ func (d *FileDeleter) MarkGeneratedFiles(scene *models.Scene) error { var files []string - thumbPath := d.Paths.Scene.GetThumbnailScreenshotPath(sceneHash) - exists, _ = fsutil.FileExists(thumbPath) - if exists { - files = append(files, thumbPath) - } - - normalPath := d.Paths.Scene.GetScreenshotPath(sceneHash) - exists, _ = fsutil.FileExists(normalPath) - if exists { - files = append(files, normalPath) - } - streamPreviewPath := d.Paths.Scene.GetVideoPreviewPath(sceneHash) exists, _ = fsutil.FileExists(streamPreviewPath) if exists { diff --git a/pkg/scene/generate/generator.go b/pkg/scene/generate/generator.go index 000082414..49568fb2a 100644 --- a/pkg/scene/generate/generator.go +++ b/pkg/scene/generate/generator.go @@ -38,9 +38,6 @@ type ScenePaths interface { GetVideoPreviewPath(checksum string) string GetWebpPreviewPath(checksum string) string - GetScreenshotPath(checksum string) string - GetThumbnailScreenshotPath(checksum string) string - GetSpriteImageFilePath(checksum string) string GetSpriteVttFilePath(checksum string) string @@ -106,6 +103,26 @@ func (g Generator) generateFile(lockCtx *fsutil.LockContext, p Paths, pattern st return nil } +// generateBytes performs a generate operation by generating a temporary file using p and pattern, returns the contents, then deletes it. +func (g Generator) generateBytes(lockCtx *fsutil.LockContext, p Paths, pattern string, generateFn generateFn) ([]byte, error) { + tmpFile, err := g.tempFile(p, pattern) // tmp output in case the process ends abruptly + if err != nil { + return nil, err + } + + tmpFn := tmpFile.Name() + defer func() { + _ = os.Remove(tmpFn) + }() + + if err := generateFn(lockCtx, tmpFn); err != nil { + return nil, err + } + + defer os.Remove(tmpFn) + return os.ReadFile(tmpFn) +} + // generate runs ffmpeg with the given args and waits for it to finish. // Returns an error if the command fails. If the command fails, the return // value will be of type *exec.ExitError. diff --git a/pkg/scene/generate/screenshot.go b/pkg/scene/generate/screenshot.go index 41ecc8fe8..917a481ef 100644 --- a/pkg/scene/generate/screenshot.go +++ b/pkg/scene/generate/screenshot.go @@ -9,8 +9,8 @@ import ( ) const ( - thumbnailWidth = 320 - thumbnailQuality = 5 + // thumbnailWidth = 320 + // thumbnailQuality = 5 screenshotQuality = 2 @@ -21,17 +21,10 @@ type ScreenshotOptions struct { At *float64 } -func (g Generator) Screenshot(ctx context.Context, input string, hash string, videoWidth int, videoDuration float64, options ScreenshotOptions) error { +func (g Generator) Screenshot(ctx context.Context, input string, videoWidth int, videoDuration float64, options ScreenshotOptions) ([]byte, error) { lockCtx := g.LockManager.ReadLock(ctx, input) defer lockCtx.Cancel() - output := g.ScenePaths.GetScreenshotPath(hash) - if !g.Overwrite { - if exists, _ := fsutil.FileExists(output); exists { - return nil - } - } - logger.Infof("Creating screenshot for %s", input) at := screenshotDurationProportion * videoDuration @@ -39,46 +32,16 @@ func (g Generator) Screenshot(ctx context.Context, input string, hash string, vi at = *options.At } - if err := g.generateFile(lockCtx, g.ScenePaths, jpgPattern, output, g.screenshot(input, screenshotOptions{ + ret, err := g.generateBytes(lockCtx, g.ScenePaths, jpgPattern, g.screenshot(input, screenshotOptions{ Time: at, Quality: screenshotQuality, // default Width is video width - })); err != nil { - return err + })) + if err != nil { + return nil, err } - logger.Debug("created screenshot: ", output) - - return nil -} - -func (g Generator) Thumbnail(ctx context.Context, input string, hash string, videoDuration float64, options ScreenshotOptions) error { - lockCtx := g.LockManager.ReadLock(ctx, input) - defer lockCtx.Cancel() - - output := g.ScenePaths.GetThumbnailScreenshotPath(hash) - if !g.Overwrite { - if exists, _ := fsutil.FileExists(output); exists { - return nil - } - } - - at := screenshotDurationProportion * videoDuration - if options.At != nil { - at = *options.At - } - - if err := g.generateFile(lockCtx, g.ScenePaths, jpgPattern, output, g.screenshot(input, screenshotOptions{ - Time: at, - Quality: thumbnailQuality, - Width: thumbnailWidth, - })); err != nil { - return err - } - - logger.Debug("created thumbnail: ", output) - - return nil + return ret, nil } type screenshotOptions struct { diff --git a/pkg/scene/merge.go b/pkg/scene/merge.go index d14f9621f..238d5233c 100644 --- a/pkg/scene/merge.go +++ b/pkg/scene/merge.go @@ -123,7 +123,7 @@ func (s *Service) mergeSceneMarkers(ctx context.Context, dest *models.Scene, src } if len(toRename) > 0 { - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { // rename the files if they exist for _, e := range toRename { srcExists, _ := fsutil.FileExists(e.src) @@ -135,8 +135,6 @@ func (s *Service) mergeSceneMarkers(ctx context.Context, dest *models.Scene, src } } } - - return nil }) } diff --git a/pkg/scene/migrate_hash.go b/pkg/scene/migrate_hash.go index 7f39d5370..09b25297f 100644 --- a/pkg/scene/migrate_hash.go +++ b/pkg/scene/migrate_hash.go @@ -16,14 +16,6 @@ func MigrateHash(p *paths.Paths, oldHash string, newHash string) { migrateSceneFiles(oldPath, newPath) scenePaths := p.Scene - oldPath = scenePaths.GetThumbnailScreenshotPath(oldHash) - newPath = scenePaths.GetThumbnailScreenshotPath(newHash) - migrateSceneFiles(oldPath, newPath) - - oldPath = scenePaths.GetScreenshotPath(oldHash) - newPath = scenePaths.GetScreenshotPath(newHash) - migrateSceneFiles(oldPath, newPath) - oldPath = scenePaths.GetVideoPreviewPath(oldHash) newPath = scenePaths.GetVideoPreviewPath(newHash) migrateSceneFiles(oldPath, newPath) diff --git a/pkg/scene/migrate_screenshots.go b/pkg/scene/migrate_screenshots.go new file mode 100644 index 000000000..94d73643f --- /dev/null +++ b/pkg/scene/migrate_screenshots.go @@ -0,0 +1,143 @@ +package scene + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/txn" +) + +type MigrateSceneScreenshotsInput struct { + DeleteFiles bool `json:"deleteFiles"` + OverwriteExisting bool `json:"overwriteExisting"` +} + +type HashFinderCoverUpdater interface { + FindByChecksum(ctx context.Context, checksum string) ([]*models.Scene, error) + FindByOSHash(ctx context.Context, oshash string) ([]*models.Scene, error) + CoverUpdater +} + +type ScreenshotMigrator struct { + Options MigrateSceneScreenshotsInput + SceneUpdater HashFinderCoverUpdater + TxnManager txn.Manager +} + +func (m *ScreenshotMigrator) MigrateScreenshots(ctx context.Context, screenshotPath string) error { + // find the scene based on the screenshot path + s, err := m.findScenes(ctx, screenshotPath) + if err != nil { + return fmt.Errorf("finding scenes for screenshot: %w", err) + } + + for _, scene := range s { + // migrate each scene in its own transaction + if err := txn.WithTxn(ctx, m.TxnManager, func(ctx context.Context) error { + return m.migrateSceneScreenshot(ctx, scene, screenshotPath) + }); err != nil { + return fmt.Errorf("migrating screenshot for scene %s: %w", scene.DisplayName(), err) + } + } + + // if deleteFiles is true, delete the file + if m.Options.DeleteFiles { + if err := os.Remove(screenshotPath); err != nil { + // log and continue + logger.Errorf("Error deleting screenshot file %s: %v", screenshotPath, err) + } else { + logger.Debugf("Deleted screenshot file %s", screenshotPath) + } + + // also delete the thumb file + thumbPath := strings.TrimSuffix(screenshotPath, ".jpg") + ".thumb.jpg" + // ignore errors for thumb files + if err := os.Remove(thumbPath); err == nil { + logger.Debugf("Deleted thumb file %s", thumbPath) + } + } + + return nil +} + +func (m *ScreenshotMigrator) findScenes(ctx context.Context, screenshotPath string) ([]*models.Scene, error) { + basename := filepath.Base(screenshotPath) + ext := filepath.Ext(basename) + basename = basename[:len(basename)-len(ext)] + + // use the basename to determine the hash type + algo := m.getHashType(basename) + + if algo == "" { + // log and return + return nil, fmt.Errorf("could not determine hash type") + } + + // use the hash type to get the scene + var ret []*models.Scene + err := txn.WithReadTxn(ctx, m.TxnManager, func(ctx context.Context) error { + var err error + + if algo == models.HashAlgorithmOshash { + // use oshash + ret, err = m.SceneUpdater.FindByOSHash(ctx, basename) + } else { + // use md5 + ret, err = m.SceneUpdater.FindByChecksum(ctx, basename) + } + + return err + }) + + return ret, err +} + +func (m *ScreenshotMigrator) getHashType(basename string) models.HashAlgorithm { + // if the basename is 16 characters long, must be oshash + if len(basename) == 16 { + return models.HashAlgorithmOshash + } + + // if its 32 characters long, must be md5 + if len(basename) == 32 { + return models.HashAlgorithmMd5 + } + + // otherwise, it's undefined + return "" +} + +func (m *ScreenshotMigrator) migrateSceneScreenshot(ctx context.Context, scene *models.Scene, screenshotPath string) error { + if !m.Options.OverwriteExisting { + // check if the scene has a cover already + hasCover, err := m.SceneUpdater.HasCover(ctx, scene.ID) + if err != nil { + return fmt.Errorf("checking for existing cover: %w", err) + } + + if hasCover { + // already has cover, just silently return + logger.Debugf("Scene %s already has a screenshot, skipping", scene.DisplayName()) + return nil + } + } + + // get the data from the file + data, err := os.ReadFile(screenshotPath) + if err != nil { + return fmt.Errorf("reading screenshot file: %w", err) + } + + if err := m.SceneUpdater.UpdateCover(ctx, scene.ID, data); err != nil { + return fmt.Errorf("updating scene screenshot: %w", err) + } + + logger.Infof("Updated screenshot for scene %s from %s", scene.DisplayName(), filepath.Base(screenshotPath)) + + return nil +} diff --git a/pkg/scene/scan.go b/pkg/scene/scan.go index 8abbfc2d3..5ccdee256 100644 --- a/pkg/scene/scan.go +++ b/pkg/scene/scan.go @@ -35,7 +35,6 @@ type ScanGenerator interface { type ScanHandler struct { CreatorUpdater CreatorUpdater - CoverGenerator CoverGenerator ScanGenerator ScanGenerator CaptionUpdater video.CaptionUpdater PluginCache *plugin.Cache @@ -48,9 +47,6 @@ func (h *ScanHandler) validate() error { if h.CreatorUpdater == nil { return errors.New("CreatorUpdater is required") } - if h.CoverGenerator == nil { - return errors.New("CoverGenerator is required") - } if h.ScanGenerator == nil { return errors.New("ScanGenerator is required") } @@ -132,20 +128,13 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File } // do this after the commit so that cover generation doesn't hold up the transaction - txn.AddPostCommitHook(ctx, func(ctx context.Context) error { + txn.AddPostCommitHook(ctx, func(ctx context.Context) { for _, s := range existing { - if err := h.CoverGenerator.GenerateCover(ctx, s, videoFile); err != nil { - // just log if cover generation fails. We can try again on rescan - logger.Errorf("Error generating cover for %s: %v", videoFile.Path, err) - } - if err := h.ScanGenerator.Generate(ctx, s, videoFile); err != nil { // just log if cover generation fails. We can try again on rescan logger.Errorf("Error generating content for %s: %v", videoFile.Path, err) } } - - return nil }) return nil diff --git a/pkg/scene/screenshot.go b/pkg/scene/screenshot.go deleted file mode 100644 index c33203327..000000000 --- a/pkg/scene/screenshot.go +++ /dev/null @@ -1,103 +0,0 @@ -package scene - -import ( - "bytes" - "context" - "image" - "image/jpeg" - "os" - - "github.com/stashapp/stash/pkg/file" - "github.com/stashapp/stash/pkg/fsutil" - "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/models/paths" - - "github.com/disintegration/imaging" - - // needed to decode other image formats - _ "image/gif" - _ "image/png" -) - -type CoverGenerator interface { - GenerateCover(ctx context.Context, scene *models.Scene, f *file.VideoFile) error -} - -type ScreenshotSetter interface { - SetScreenshot(scene *models.Scene, imageData []byte) error -} - -type PathsCoverSetter struct { - Paths *paths.Paths - FileNamingAlgorithm models.HashAlgorithm -} - -func (ss *PathsCoverSetter) SetScreenshot(scene *models.Scene, imageData []byte) error { - // don't set where scene has no file - if scene.Path == "" { - return nil - } - checksum := scene.GetHash(ss.FileNamingAlgorithm) - return SetScreenshot(ss.Paths, checksum, imageData) -} - -func writeImage(path string, imageData []byte) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - _, err = f.Write(imageData) - return err -} - -func writeThumbnail(path string, thumbnail image.Image) error { - f, err := os.Create(path) - if err != nil { - return err - } - defer f.Close() - - return jpeg.Encode(f, thumbnail, nil) -} - -func SetScreenshot(paths *paths.Paths, checksum string, imageData []byte) error { - thumbPath := paths.Scene.GetThumbnailScreenshotPath(checksum) - normalPath := paths.Scene.GetScreenshotPath(checksum) - - img, _, err := image.Decode(bytes.NewReader(imageData)) - if err != nil { - return err - } - - // resize to 320 width maintaining aspect ratio, for the thumbnail - const width = 320 - origWidth := img.Bounds().Max.X - origHeight := img.Bounds().Max.Y - height := width / origWidth * origHeight - - thumbnail := imaging.Resize(img, width, height, imaging.Lanczos) - err = writeThumbnail(thumbPath, thumbnail) - if err != nil { - return err - } - - err = writeImage(normalPath, imageData) - - return err -} - -func (s *Service) GetCover(ctx context.Context, scene *models.Scene) ([]byte, error) { - if scene.Path != "" { - filepath := s.Paths.Scene.GetScreenshotPath(scene.GetHash(s.Config.GetVideoFileNamingAlgorithm())) - - // fall back to the scene image blob if the file isn't present - screenshotExists, _ := fsutil.FileExists(filepath) - if screenshotExists { - return os.ReadFile(filepath) - } - } - - return s.Repository.GetCover(ctx, scene.ID) -} diff --git a/pkg/scene/service.go b/pkg/scene/service.go index c162858af..a3d01dd3d 100644 --- a/pkg/scene/service.go +++ b/pkg/scene/service.go @@ -22,6 +22,7 @@ type Creator interface { } type CoverUpdater interface { + HasCover(ctx context.Context, sceneID int) (bool, error) UpdateCover(ctx context.Context, sceneID int, cover []byte) error } diff --git a/pkg/scene/update.go b/pkg/scene/update.go index c38597da7..e3f3e252b 100644 --- a/pkg/scene/update.go +++ b/pkg/scene/update.go @@ -46,7 +46,7 @@ func (u *UpdateSet) IsEmpty() bool { // Update updates a scene by updating the fields in the Partial field, then // updates non-nil relationships. Returns an error if there is no work to // be done. -func (u *UpdateSet) Update(ctx context.Context, qb Updater, screenshotSetter ScreenshotSetter) (*models.Scene, error) { +func (u *UpdateSet) Update(ctx context.Context, qb Updater) (*models.Scene, error) { if u.IsEmpty() { return nil, ErrEmptyUpdater } @@ -64,10 +64,6 @@ func (u *UpdateSet) Update(ctx context.Context, qb Updater, screenshotSetter Scr if err := qb.UpdateCover(ctx, u.ID, u.CoverImage); err != nil { return nil, fmt.Errorf("error updating scene cover: %w", err) } - - if err := screenshotSetter.SetScreenshot(ret, u.CoverImage); err != nil { - return nil, fmt.Errorf("error setting scene screenshot: %w", err) - } } return ret, nil diff --git a/pkg/scene/update_test.go b/pkg/scene/update_test.go index ffd84f00c..c89be66f3 100644 --- a/pkg/scene/update_test.go +++ b/pkg/scene/update_test.go @@ -93,12 +93,6 @@ func TestUpdater_IsEmpty(t *testing.T) { } } -type mockScreenshotSetter struct{} - -func (s *mockScreenshotSetter) SetScreenshot(scene *models.Scene, imageData []byte) error { - return nil -} - func TestUpdater_Update(t *testing.T) { const ( sceneID = iota + 1 @@ -210,7 +204,7 @@ func TestUpdater_Update(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := tt.u.Update(ctx, &qb, &mockScreenshotSetter{}) + got, err := tt.u.Update(ctx, &qb) if (err != nil) != tt.wantErr { t.Errorf("Updater.Update() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 8265d8050..9d8e65d0c 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -932,7 +932,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo } draft.Tags = tags - if cover != nil { + if len(cover) > 0 { image = bytes.NewReader(cover) } diff --git a/pkg/sqlite/anonymise.go b/pkg/sqlite/anonymise.go index abf919f15..c16d1160d 100644 --- a/pkg/sqlite/anonymise.go +++ b/pkg/sqlite/anonymise.go @@ -70,6 +70,11 @@ func (db *Anonymiser) Anonymise(ctx context.Context) error { return nil } +func (db *Anonymiser) truncateColumn(tableName string, column string) error { + _, err := db.db.Exec("UPDATE " + tableName + " SET " + column + " = NULL") + return err +} + func (db *Anonymiser) truncateTable(tableName string) error { _, err := db.db.Exec("DELETE FROM " + tableName) return err @@ -77,11 +82,14 @@ func (db *Anonymiser) truncateTable(tableName string) error { func (db *Anonymiser) deleteBlobs() error { return utils.Do([]func() error{ - func() error { return db.truncateTable("scenes_cover") }, - func() error { return db.truncateTable("movies_images") }, - func() error { return db.truncateTable("performers_image") }, - func() error { return db.truncateTable("studios_image") }, - func() error { return db.truncateTable("tags_image") }, + func() error { return db.truncateColumn("tags", "image_blob") }, + func() error { return db.truncateColumn("studios", "image_blob") }, + func() error { return db.truncateColumn("performers", "image_blob") }, + func() error { return db.truncateColumn("scenes", "cover_blob") }, + func() error { return db.truncateColumn("movies", "front_image_blob") }, + func() error { return db.truncateColumn("movies", "back_image_blob") }, + + func() error { return db.truncateTable("blobs") }, }) } diff --git a/pkg/sqlite/blob.go b/pkg/sqlite/blob.go new file mode 100644 index 000000000..96c597fa5 --- /dev/null +++ b/pkg/sqlite/blob.go @@ -0,0 +1,382 @@ +package sqlite + +import ( + "context" + "database/sql" + "errors" + "fmt" + "io/fs" + + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" + "github.com/jmoiron/sqlx" + "github.com/mattn/go-sqlite3" + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/hash/md5" + "github.com/stashapp/stash/pkg/sqlite/blob" + "github.com/stashapp/stash/pkg/utils" + "gopkg.in/guregu/null.v4" +) + +const ( + blobTable = "blobs" + blobChecksumColumn = "checksum" +) + +type BlobStoreOptions struct { + // UseFilesystem should be true if blob data should be stored in the filesystem + UseFilesystem bool + // UseDatabase should be true if blob data should be stored in the database + UseDatabase bool + // Path is the filesystem path to use for storing blobs + Path string +} + +type BlobStore struct { + repository + + tableMgr *table + + fsStore *blob.FilesystemStore + options BlobStoreOptions +} + +func NewBlobStore(options BlobStoreOptions) *BlobStore { + return &BlobStore{ + repository: repository{ + tableName: blobTable, + idColumn: blobChecksumColumn, + }, + + tableMgr: blobTableMgr, + + fsStore: blob.NewFilesystemStore(options.Path, &file.OsFS{}), + options: options, + } +} + +type blobRow struct { + Checksum string `db:"checksum"` + Blob []byte `db:"blob"` +} + +func (qb *BlobStore) table() exp.IdentifierExpression { + return qb.tableMgr.table +} + +func (qb *BlobStore) Count(ctx context.Context) (int, error) { + table := qb.table() + q := dialect.From(table).Select(goqu.COUNT(table.Col(blobChecksumColumn))) + + var ret int + if err := querySimple(ctx, q, &ret); err != nil { + return 0, err + } + + return ret, nil +} + +// Write stores the data and its checksum in enabled stores. +// Always writes at least the checksum to the database. +func (qb *BlobStore) Write(ctx context.Context, data []byte) (string, error) { + if !qb.options.UseDatabase && !qb.options.UseFilesystem { + panic("no blob store configured") + } + + if len(data) == 0 { + return "", fmt.Errorf("cannot write empty data") + } + + checksum := md5.FromBytes(data) + + // only write blob to the database if UseDatabase is true + // always at least write the checksum + var storedData []byte + if qb.options.UseDatabase { + storedData = data + } + + if err := qb.write(ctx, checksum, storedData); err != nil { + return "", fmt.Errorf("writing to database: %w", err) + } + + if qb.options.UseFilesystem { + if err := qb.fsStore.Write(ctx, checksum, data); err != nil { + return "", fmt.Errorf("writing to filesystem: %w", err) + } + } + + return checksum, nil +} + +func (qb *BlobStore) write(ctx context.Context, checksum string, data []byte) error { + table := qb.table() + q := dialect.Insert(table).Prepared(true).Rows(blobRow{ + Checksum: checksum, + Blob: data, + }).OnConflict(goqu.DoNothing()) + + _, err := exec(ctx, q) + if err != nil { + return fmt.Errorf("inserting into %s: %w", table, err) + } + + return nil +} + +func (qb *BlobStore) update(ctx context.Context, checksum string, data []byte) error { + table := qb.table() + q := dialect.Update(table).Prepared(true).Set(goqu.Record{ + "blob": data, + }).Where(goqu.C(blobChecksumColumn).Eq(checksum)) + + _, err := exec(ctx, q) + if err != nil { + return fmt.Errorf("updating %s: %w", table, err) + } + + return nil +} + +type ChecksumNotFoundError struct { + Checksum string +} + +func (e *ChecksumNotFoundError) Error() string { + return fmt.Sprintf("checksum %s does not exist", e.Checksum) +} + +type ChecksumBlobNotExistError struct { + Checksum string +} + +func (e *ChecksumBlobNotExistError) Error() string { + return fmt.Sprintf("blob for checksum %s does not exist", e.Checksum) +} + +func (qb *BlobStore) readSQL(ctx context.Context, querySQL string, args ...interface{}) ([]byte, string, error) { + if !qb.options.UseDatabase && !qb.options.UseFilesystem { + panic("no blob store configured") + } + + // always try to get from the database first, even if set to use filesystem + var row blobRow + found := false + const single = true + if err := qb.queryFunc(ctx, querySQL, args, single, func(r *sqlx.Rows) error { + found = true + if err := r.StructScan(&row); err != nil { + return err + } + + return nil + }); err != nil { + return nil, "", fmt.Errorf("reading from database: %w", err) + } + + if !found { + // not found in the database - does not exist + return nil, "", nil + } + + checksum := row.Checksum + + if row.Blob != nil { + return row.Blob, checksum, nil + } + + // don't use the filesystem if not configured to do so + if qb.options.UseFilesystem { + ret, err := qb.fsStore.Read(ctx, checksum) + if err == nil { + return ret, checksum, nil + } + + if !errors.Is(err, fs.ErrNotExist) { + return nil, checksum, fmt.Errorf("reading from filesystem: %w", err) + } + } + + return nil, checksum, &ChecksumBlobNotExistError{ + Checksum: checksum, + } +} + +// Read reads the data from the database or filesystem, depending on which is enabled. +func (qb *BlobStore) Read(ctx context.Context, checksum string) ([]byte, error) { + if !qb.options.UseDatabase && !qb.options.UseFilesystem { + panic("no blob store configured") + } + + // always try to get from the database first, even if set to use filesystem + ret, err := qb.readFromDatabase(ctx, checksum) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("reading from database: %w", err) + } + + // not found in the database - does not exist + return nil, &ChecksumNotFoundError{ + Checksum: checksum, + } + } + + if ret != nil { + return ret, nil + } + + // don't use the filesystem if not configured to do so + if qb.options.UseFilesystem { + ret, err := qb.fsStore.Read(ctx, checksum) + if err == nil { + return ret, nil + } + + if !errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("reading from filesystem: %w", err) + } + } + + // blob not found - should not happen + return nil, &ChecksumBlobNotExistError{ + Checksum: checksum, + } +} + +func (qb *BlobStore) readFromDatabase(ctx context.Context, checksum string) ([]byte, error) { + q := dialect.From(qb.table()).Select(qb.table().All()).Where(qb.tableMgr.byID(checksum)) + + var row blobRow + const single = true + if err := queryFunc(ctx, q, single, func(r *sqlx.Rows) error { + if err := r.StructScan(&row); err != nil { + return err + } + + return nil + }); err != nil { + return nil, fmt.Errorf("querying %s: %w", qb.table(), err) + } + + return row.Blob, nil +} + +// Delete marks a checksum as no longer in use by a single reference. +// If no references remain, the blob is deleted from the database and filesystem. +func (qb *BlobStore) Delete(ctx context.Context, checksum string) error { + // try to delete the blob from the database + if err := qb.delete(ctx, checksum); err != nil { + if qb.isConstraintError(err) { + // blob is still referenced - do not delete + return nil + } + + // unexpected error + return fmt.Errorf("deleting from database: %w", err) + } + + // blob was deleted from the database - delete from filesystem if enabled + if qb.options.UseFilesystem { + if err := qb.fsStore.Delete(ctx, checksum); err != nil { + return fmt.Errorf("deleting from filesystem: %w", err) + } + } + + return nil +} + +func (qb *BlobStore) isConstraintError(err error) bool { + var sqliteError sqlite3.Error + if errors.As(err, &sqliteError) { + return sqliteError.Code == sqlite3.ErrConstraint + } + return false +} + +func (qb *BlobStore) delete(ctx context.Context, checksum string) error { + table := qb.table() + + q := dialect.Delete(table).Where(goqu.C(blobChecksumColumn).Eq(checksum)) + + _, err := exec(ctx, q) + if err != nil { + return fmt.Errorf("deleting from %s: %w", table, err) + } + + return nil +} + +type blobJoinQueryBuilder struct { + repository + blobStore *BlobStore + + joinTable string +} + +func (qb *blobJoinQueryBuilder) GetImage(ctx context.Context, id int, blobCol string) ([]byte, error) { + sqlQuery := utils.StrFormat(` +SELECT blobs.checksum, blobs.blob FROM {joinTable} INNER JOIN blobs ON {joinTable}.{joinCol} = blobs.checksum +WHERE {joinTable}.id = ? +`, utils.StrFormatMap{ + "joinTable": qb.joinTable, + "joinCol": blobCol, + }) + + ret, _, err := qb.blobStore.readSQL(ctx, sqlQuery, id) + return ret, err +} + +func (qb *blobJoinQueryBuilder) UpdateImage(ctx context.Context, id int, blobCol string, image []byte) error { + if len(image) == 0 { + return qb.DestroyImage(ctx, id, blobCol) + } + checksum, err := qb.blobStore.Write(ctx, image) + if err != nil { + return err + } + + sqlQuery := fmt.Sprintf("UPDATE %s SET %s = ? WHERE id = ?", qb.joinTable, blobCol) + _, err = qb.tx.Exec(ctx, sqlQuery, checksum, id) + return err +} + +func (qb *blobJoinQueryBuilder) DestroyImage(ctx context.Context, id int, blobCol string) error { + sqlQuery := utils.StrFormat(` +SELECT {joinTable}.{joinCol} FROM {joinTable} WHERE {joinTable}.id = ? +`, utils.StrFormatMap{ + "joinTable": qb.joinTable, + "joinCol": blobCol, + }) + + var checksum null.String + err := qb.repository.querySimple(ctx, sqlQuery, []interface{}{id}, &checksum) + if err != nil { + return err + } + + if !checksum.Valid { + // no image to delete + return nil + } + + updateQuery := fmt.Sprintf("UPDATE %s SET %s = NULL WHERE id = ?", qb.joinTable, blobCol) + if _, err = qb.tx.Exec(ctx, updateQuery, id); err != nil { + return err + } + + return qb.blobStore.Delete(ctx, checksum.String) +} + +func (qb *blobJoinQueryBuilder) HasImage(ctx context.Context, id int, blobCol string) (bool, error) { + stmt := utils.StrFormat("SELECT COUNT(*) as count FROM (SELECT {joinCol} FROM {joinTable} WHERE id = ? AND {joinCol} IS NOT NULL LIMIT 1)", utils.StrFormatMap{ + "joinTable": qb.joinTable, + "joinCol": blobCol, + }) + + c, err := qb.runCountQuery(ctx, stmt, []interface{}{id}) + if err != nil { + return false, err + } + + return c == 1, nil +} diff --git a/pkg/sqlite/blob/fs.go b/pkg/sqlite/blob/fs.go new file mode 100644 index 000000000..37ec4b064 --- /dev/null +++ b/pkg/sqlite/blob/fs.go @@ -0,0 +1,108 @@ +package blob + +import ( + "bytes" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/fsutil" +) + +const ( + blobsDirDepth int = 2 + blobsDirLength int = 2 // thumbDirDepth * thumbDirLength must be smaller than the length of checksum +) + +type FS interface { + Create(name string) (*os.File, error) + MkdirAll(path string, perm fs.FileMode) error + Open(name string) (fs.ReadDirFile, error) + Remove(name string) error + + file.RenamerRemover +} + +type FilesystemStore struct { + deleter *file.Deleter + path string + fs FS +} + +func NewFilesystemStore(path string, fs FS) *FilesystemStore { + deleter := &file.Deleter{ + RenamerRemover: fs, + } + + return &FilesystemStore{ + deleter: deleter, + path: path, + fs: fs, + } +} + +func (s *FilesystemStore) checksumToPath(checksum string) string { + return filepath.Join(s.path, fsutil.GetIntraDir(checksum, blobsDirDepth, blobsDirLength), checksum) +} + +func (s *FilesystemStore) Write(ctx context.Context, checksum string, data []byte) error { + if s.path == "" { + return fmt.Errorf("no path set") + } + + fn := s.checksumToPath(checksum) + + // create the directory if it doesn't exist + if err := s.fs.MkdirAll(filepath.Dir(fn), 0755); err != nil { + return fmt.Errorf("creating directory %q: %w", filepath.Dir(fn), err) + } + + out, err := s.fs.Create(fn) + if err != nil { + return fmt.Errorf("creating file %q: %w", fn, err) + } + + r := bytes.NewReader(data) + + if _, err = io.Copy(out, r); err != nil { + return fmt.Errorf("writing file %q: %w", fn, err) + } + + return nil +} + +func (s *FilesystemStore) Read(ctx context.Context, checksum string) ([]byte, error) { + if s.path == "" { + return nil, fmt.Errorf("no path set") + } + + fn := s.checksumToPath(checksum) + f, err := s.fs.Open(fn) + if err != nil { + return nil, fmt.Errorf("opening file %q: %w", fn, err) + } + + defer f.Close() + + return io.ReadAll(f) +} + +func (s *FilesystemStore) Delete(ctx context.Context, checksum string) error { + if s.path == "" { + return fmt.Errorf("no path set") + } + + s.deleter.RegisterHooks(ctx) + + fn := s.checksumToPath(checksum) + + if err := s.deleter.Files([]string{fn}); err != nil { + return fmt.Errorf("deleting file %q: %w", fn, err) + } + + return nil +} diff --git a/pkg/sqlite/blob_migrate.go b/pkg/sqlite/blob_migrate.go new file mode 100644 index 000000000..e121d0792 --- /dev/null +++ b/pkg/sqlite/blob_migrate.go @@ -0,0 +1,116 @@ +package sqlite + +import ( + "context" + "fmt" + + "github.com/jmoiron/sqlx" +) + +func (qb *BlobStore) FindBlobs(ctx context.Context, n uint, lastChecksum string) ([]string, error) { + table := qb.table() + q := dialect.From(table).Select(table.Col(blobChecksumColumn)).Order(table.Col(blobChecksumColumn).Asc()).Limit(n) + + if lastChecksum != "" { + q = q.Where(table.Col(blobChecksumColumn).Gt(lastChecksum)) + } + + const single = false + var checksums []string + if err := queryFunc(ctx, q, single, func(rows *sqlx.Rows) error { + var checksum string + if err := rows.Scan(&checksum); err != nil { + return err + } + checksums = append(checksums, checksum) + return nil + }); err != nil { + return nil, err + } + + return checksums, nil +} + +// MigrateBlob migrates a blob from the filesystem to the database, or vice versa. +// The target is determined by the UseDatabase and UseFilesystem options. +// If deleteOld is true, the blob is deleted from the source after migration. +func (qb *BlobStore) MigrateBlob(ctx context.Context, checksum string, deleteOld bool) error { + if !qb.options.UseDatabase && !qb.options.UseFilesystem { + panic("no blob store configured") + } + + if qb.options.UseDatabase && qb.options.UseFilesystem { + panic("both filesystem and database configured") + } + + if qb.options.Path == "" { + panic("no blob path configured") + } + + if qb.options.UseDatabase { + return qb.migrateBlobDatabase(ctx, checksum, deleteOld) + } + + return qb.migrateBlobFilesystem(ctx, checksum, deleteOld) +} + +// migrateBlobDatabase migrates a blob from the filesystem to the database +func (qb *BlobStore) migrateBlobDatabase(ctx context.Context, checksum string, deleteOld bool) error { + // ignore if the blob is already present in the database + // (still delete the old data if requested) + existing, err := qb.readFromDatabase(ctx, checksum) + if err != nil { + return fmt.Errorf("reading from database: %w", err) + } + + if len(existing) == 0 { + // find the blob in the filesystem + blob, err := qb.fsStore.Read(ctx, checksum) + if err != nil { + return fmt.Errorf("reading from filesystem: %w", err) + } + + // write the blob to the database + if err := qb.update(ctx, checksum, blob); err != nil { + return fmt.Errorf("writing to database: %w", err) + } + } + + if deleteOld { + // delete the blob from the filesystem after commit + if err := qb.fsStore.Delete(ctx, checksum); err != nil { + return fmt.Errorf("deleting from filesystem: %w", err) + } + } + + return nil +} + +// migrateBlobFilesystem migrates a blob from the database to the filesystem +func (qb *BlobStore) migrateBlobFilesystem(ctx context.Context, checksum string, deleteOld bool) error { + // find the blob in the database + blob, err := qb.readFromDatabase(ctx, checksum) + if err != nil { + return fmt.Errorf("reading from database: %w", err) + } + + if len(blob) == 0 { + // it's possible that the blob is already present in the filesystem + // just ignore + return nil + } + + // write the blob to the filesystem + if err := qb.fsStore.Write(ctx, checksum, blob); err != nil { + return fmt.Errorf("writing to filesystem: %w", err) + } + + if deleteOld { + // delete the blob from the database row + if err := qb.update(ctx, checksum, nil); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/sqlite/blob_test.go b/pkg/sqlite/blob_test.go new file mode 100644 index 000000000..4c6e0ccc2 --- /dev/null +++ b/pkg/sqlite/blob_test.go @@ -0,0 +1,45 @@ +//go:build integration +// +build integration + +package sqlite_test + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +type updateImageFunc func(ctx context.Context, id int, image []byte) error +type getImageFunc func(ctx context.Context, movieID int) ([]byte, error) + +func testUpdateImage(t *testing.T, ctx context.Context, id int, updateFn updateImageFunc, getFn getImageFunc) error { + image := []byte("image") + err := updateFn(ctx, id, image) + if err != nil { + return fmt.Errorf("Error updating performer image: %s", err.Error()) + } + + // ensure image set + storedImage, err := getFn(ctx, id) + if err != nil { + return fmt.Errorf("Error getting image: %s", err.Error()) + } + assert.Equal(t, storedImage, image) + + // set nil image + err = updateFn(ctx, id, nil) + if err != nil { + return fmt.Errorf("error setting nil image: %w", err) + } + + // ensure image null + storedImage, err = getFn(ctx, id) + if err != nil { + return fmt.Errorf("Error getting image: %s", err.Error()) + } + assert.Nil(t, storedImage) + + return nil +} diff --git a/pkg/sqlite/database.go b/pkg/sqlite/database.go index c107986c1..ee3fa5c3d 100644 --- a/pkg/sqlite/database.go +++ b/pkg/sqlite/database.go @@ -32,7 +32,7 @@ const ( dbConnTimeout = 30 ) -var appSchemaVersion uint = 44 +var appSchemaVersion uint = 45 //go:embed migrations/*.sql var migrationsBox embed.FS @@ -64,12 +64,16 @@ func (e *MismatchedSchemaVersionError) Error() string { } type Database struct { + Blobs *BlobStore File *FileStore Folder *FolderStore Image *ImageStore Gallery *GalleryStore Scene *SceneStore Performer *PerformerStore + Studio *studioQueryBuilder + Tag *tagQueryBuilder + Movie *movieQueryBuilder db *sqlx.DB dbPath string @@ -82,20 +86,29 @@ type Database struct { func NewDatabase() *Database { fileStore := NewFileStore() folderStore := NewFolderStore() + blobStore := NewBlobStore(BlobStoreOptions{}) ret := &Database{ + Blobs: blobStore, File: fileStore, Folder: folderStore, - Scene: NewSceneStore(fileStore), + Scene: NewSceneStore(fileStore, blobStore), Image: NewImageStore(fileStore), Gallery: NewGalleryStore(fileStore, folderStore), - Performer: NewPerformerStore(), + Performer: NewPerformerStore(blobStore), + Studio: NewStudioReaderWriter(blobStore), + Tag: NewTagReaderWriter(blobStore), + Movie: NewMovieReaderWriter(blobStore), lockChan: make(chan struct{}, 1), } return ret } +func (db *Database) SetBlobStoreOptions(options BlobStoreOptions) { + *db.Blobs = *NewBlobStore(options) +} + // Ready returns an error if the database is not ready to begin transactions. func (db *Database) Ready() error { if db.db == nil { @@ -433,6 +446,12 @@ func (db *Database) optimise() { } } +// Vacuum runs a VACUUM on the database, rebuilding the database file into a minimal amount of disk space. +func (db *Database) Vacuum(ctx context.Context) error { + _, err := db.db.ExecContext(ctx, "VACUUM") + return err +} + func (db *Database) runCustomMigrations(ctx context.Context, fns []customMigrationFunc) error { for _, fn := range fns { if err := db.runCustomMigration(ctx, fn); err != nil { diff --git a/pkg/sqlite/migrations/45_blobs.up.sql b/pkg/sqlite/migrations/45_blobs.up.sql new file mode 100644 index 000000000..ea62fe5bc --- /dev/null +++ b/pkg/sqlite/migrations/45_blobs.up.sql @@ -0,0 +1,19 @@ +CREATE TABLE `blobs` ( + `checksum` varchar(255) NOT NULL PRIMARY KEY, + `blob` blob +); + +ALTER TABLE `tags` ADD COLUMN `image_blob` varchar(255) REFERENCES `blobs`(`checksum`); +ALTER TABLE `studios` ADD COLUMN `image_blob` varchar(255) REFERENCES `blobs`(`checksum`); +ALTER TABLE `performers` ADD COLUMN `image_blob` varchar(255) REFERENCES `blobs`(`checksum`); +ALTER TABLE `scenes` ADD COLUMN `cover_blob` varchar(255) REFERENCES `blobs`(`checksum`); + +ALTER TABLE `movies` ADD COLUMN `front_image_blob` varchar(255) REFERENCES `blobs`(`checksum`); +ALTER TABLE `movies` ADD COLUMN `back_image_blob` varchar(255) REFERENCES `blobs`(`checksum`); + +-- performed in the post-migration +-- DROP TABLE `tags_image`; +-- DROP TABLE `studios_image`; +-- DROP TABLE `performers_image`; +-- DROP TABLE `scenes_cover`; +-- DROP TABLE `movies_images`; diff --git a/pkg/sqlite/migrations/45_postmigrate.go b/pkg/sqlite/migrations/45_postmigrate.go new file mode 100644 index 000000000..8618838a2 --- /dev/null +++ b/pkg/sqlite/migrations/45_postmigrate.go @@ -0,0 +1,286 @@ +package migrations + +import ( + "context" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/stashapp/stash/internal/manager/config" + "github.com/stashapp/stash/pkg/hash/md5" + "github.com/stashapp/stash/pkg/logger" + "github.com/stashapp/stash/pkg/sqlite" + "github.com/stashapp/stash/pkg/utils" +) + +type schema45Migrator struct { + migrator + hasBlobs bool +} + +func post45(ctx context.Context, db *sqlx.DB) error { + logger.Info("Running post-migration for schema version 45") + + m := schema45Migrator{ + migrator: migrator{ + db: db, + }, + } + + if err := m.migrateImagesTable(ctx, migrateImagesTableOptions{ + joinTable: "tags_image", + joinIDCol: "tag_id", + destTable: "tags", + cols: []migrateImageToBlobOptions{ + { + joinImageCol: "image", + destCol: "image_blob", + }, + }, + }); err != nil { + return err + } + + if err := m.migrateImagesTable(ctx, migrateImagesTableOptions{ + joinTable: "studios_image", + joinIDCol: "studio_id", + destTable: "studios", + cols: []migrateImageToBlobOptions{ + { + joinImageCol: "image", + destCol: "image_blob", + }, + }, + }); err != nil { + return err + } + + if err := m.migrateImagesTable(ctx, migrateImagesTableOptions{ + joinTable: "performers_image", + joinIDCol: "performer_id", + destTable: "performers", + cols: []migrateImageToBlobOptions{ + { + joinImageCol: "image", + destCol: "image_blob", + }, + }, + }); err != nil { + return err + } + + if err := m.migrateImagesTable(ctx, migrateImagesTableOptions{ + joinTable: "scenes_cover", + joinIDCol: "scene_id", + destTable: "scenes", + cols: []migrateImageToBlobOptions{ + { + joinImageCol: "cover", + destCol: "cover_blob", + }, + }, + }); err != nil { + return err + } + + if err := m.migrateImagesTable(ctx, migrateImagesTableOptions{ + joinTable: "movies_images", + joinIDCol: "movie_id", + destTable: "movies", + cols: []migrateImageToBlobOptions{ + { + joinImageCol: "front_image", + destCol: "front_image_blob", + }, + { + joinImageCol: "back_image", + destCol: "back_image_blob", + }, + }, + }); err != nil { + return err + } + + tablesToDrop := []string{ + "tags_image", + "studios_image", + "performers_image", + "scenes_cover", + "movies_images", + } + + for _, table := range tablesToDrop { + if err := m.dropTable(ctx, table); err != nil { + return err + } + } + + if err := m.migrateConfig(ctx); err != nil { + return err + } + + return nil +} + +type migrateImageToBlobOptions struct { + joinImageCol string + destCol string +} + +type migrateImagesTableOptions struct { + joinTable string + joinIDCol string + destTable string + cols []migrateImageToBlobOptions +} + +func (o migrateImagesTableOptions) selectColumns() string { + var cols []string + for _, c := range o.cols { + cols = append(cols, "`"+c.joinImageCol+"`") + } + + return strings.Join(cols, ", ") +} + +func (m *schema45Migrator) migrateImagesTable(ctx context.Context, options migrateImagesTableOptions) error { + logger.Infof("Moving %s to blobs table", options.joinTable) + + const ( + limit = 1000 + logEvery = 10000 + ) + + count := 0 + + for { + gotSome := false + + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + query := fmt.Sprintf("SELECT %s, %s FROM `%s`", options.joinIDCol, options.selectColumns(), options.joinTable) + + query += fmt.Sprintf(" LIMIT %d", limit) + + rows, err := m.db.Query(query) + if err != nil { + return err + } + defer rows.Close() + + for rows.Next() { + m.hasBlobs = true + + var id int + + result := make([]interface{}, len(options.cols)+1) + result[0] = &id + for i := range options.cols { + v := []byte{} + result[i+1] = &v + } + + err := rows.Scan(result...) + if err != nil { + return err + } + + gotSome = true + count++ + + for i, col := range options.cols { + image := result[i+1].(*[]byte) + + if len(*image) > 0 { + if err := m.insertImage(*image, id, options.destTable, col.destCol); err != nil { + return err + } + } + } + + // delete the row from the join table so we don't process it again + deleteSQL := utils.StrFormat("DELETE FROM `{joinTable}` WHERE `{joinIDCol}` = ?", utils.StrFormatMap{ + "joinTable": options.joinTable, + "joinIDCol": options.joinIDCol, + }) + if _, err := m.db.Exec(deleteSQL, id); err != nil { + return err + } + } + + return rows.Err() + }); err != nil { + return err + } + + if !gotSome { + break + } + + if count%logEvery == 0 { + logger.Infof("Migrated %d images", count) + } + } + + return nil +} + +func (m *schema45Migrator) insertImage(data []byte, id int, destTable string, destCol string) error { + // calculate checksum and insert into blobs table + checksum := md5.FromBytes(data) + + if _, err := m.db.Exec("INSERT INTO `blobs` (`checksum`, `blob`) VALUES (?, ?) ON CONFLICT DO NOTHING", checksum, data); err != nil { + return err + } + + // set the tag image checksum + updateSQL := utils.StrFormat("UPDATE `{destTable}` SET `{destCol}` = ? WHERE `id` = ?", utils.StrFormatMap{ + "destTable": destTable, + "destCol": destCol, + }) + if _, err := m.db.Exec(updateSQL, checksum, id); err != nil { + return err + } + + return nil +} + +func (m *schema45Migrator) dropTable(ctx context.Context, table string) error { + if err := m.withTxn(ctx, func(tx *sqlx.Tx) error { + logger.Debugf("Dropping %s", table) + _, err := m.db.Exec(fmt.Sprintf("DROP TABLE `%s`", table)) + return err + }); err != nil { + return err + } + + return nil +} + +func (m *schema45Migrator) migrateConfig(ctx context.Context) error { + c := config.GetInstance() + + // if we don't have blobs, and storage is already set, then don't overwrite + if !m.hasBlobs && c.GetBlobsStorage().IsValid() { + logger.Infof("Blobs storage already set, not overwriting") + return nil + } + + // if we have blobs in the database, then default to database storage + // otherwise default to filesystem storage + defaultStorage := config.BlobStorageTypeFilesystem + if m.hasBlobs || c.GetBlobsPath() == "" { + defaultStorage = config.BlobStorageTypeDatabase + } + + logger.Infof("Setting blobs storage to %s", defaultStorage.String()) + c.Set(config.BlobsStorage, defaultStorage) + if err := c.Write(); err != nil { + logger.Errorf("Error while writing configuration file: %s", err.Error()) + } + + return nil +} + +func init() { + sqlite.RegisterPostMigration(45, post45) +} diff --git a/pkg/sqlite/movies.go b/pkg/sqlite/movies.go index 5d9d5cb36..2fa429546 100644 --- a/pkg/sqlite/movies.go +++ b/pkg/sqlite/movies.go @@ -12,18 +12,30 @@ import ( "github.com/stashapp/stash/pkg/sliceutil/intslice" ) -const movieTable = "movies" -const movieIDColumn = "movie_id" +const ( + movieTable = "movies" + movieIDColumn = "movie_id" + + movieFrontImageBlobColumn = "front_image_blob" + movieBackImageBlobColumn = "back_image_blob" +) type movieQueryBuilder struct { repository + blobJoinQueryBuilder } -var MovieReaderWriter = &movieQueryBuilder{ - repository{ - tableName: movieTable, - idColumn: idColumn, - }, +func NewMovieReaderWriter(blobStore *BlobStore) *movieQueryBuilder { + return &movieQueryBuilder{ + repository{ + tableName: movieTable, + idColumn: idColumn, + }, + blobJoinQueryBuilder{ + blobStore: blobStore, + joinTable: movieTable, + }, + } } func (qb *movieQueryBuilder) Create(ctx context.Context, newObject models.Movie) (*models.Movie, error) { @@ -54,6 +66,11 @@ func (qb *movieQueryBuilder) UpdateFull(ctx context.Context, updatedObject model } func (qb *movieQueryBuilder) Destroy(ctx context.Context, id int) error { + // must handle image checksums manually + if err := qb.destroyImages(ctx, id); err != nil { + return err + } + return qb.destroyExisting(ctx, []int{id}) } @@ -209,11 +226,9 @@ func movieIsMissingCriterionHandler(qb *movieQueryBuilder, isMissing *string) cr if isMissing != nil && *isMissing != "" { switch *isMissing { case "front_image": - f.addLeftJoin("movies_images", "", "movies_images.movie_id = movies.id") - f.addWhere("movies_images.front_image IS NULL") + f.addWhere("movies.front_image_blob IS NULL") case "back_image": - f.addLeftJoin("movies_images", "", "movies_images.movie_id = movies.id") - f.addWhere("movies_images.back_image IS NULL") + f.addWhere("movies.back_image_blob IS NULL") case "scenes": f.addLeftJoin("movies_scenes", "", "movies_scenes.movie_id = movies.id") f.addWhere("movies_scenes.scene_id IS NULL") @@ -322,39 +337,31 @@ func (qb *movieQueryBuilder) queryMovies(ctx context.Context, query string, args return []*models.Movie(ret), nil } -func (qb *movieQueryBuilder) UpdateImages(ctx context.Context, movieID int, frontImage []byte, backImage []byte) error { - // Delete the existing cover and then create new - if err := qb.DestroyImages(ctx, movieID); err != nil { - return err - } - - _, err := qb.tx.Exec(ctx, - `INSERT INTO movies_images (movie_id, front_image, back_image) VALUES (?, ?, ?)`, - movieID, - frontImage, - backImage, - ) - - return err +func (qb *movieQueryBuilder) UpdateFrontImage(ctx context.Context, movieID int, frontImage []byte) error { + return qb.UpdateImage(ctx, movieID, movieFrontImageBlobColumn, frontImage) } -func (qb *movieQueryBuilder) DestroyImages(ctx context.Context, movieID int) error { - // Delete the existing joins - _, err := qb.tx.Exec(ctx, "DELETE FROM movies_images WHERE movie_id = ?", movieID) - if err != nil { +func (qb *movieQueryBuilder) UpdateBackImage(ctx context.Context, movieID int, backImage []byte) error { + return qb.UpdateImage(ctx, movieID, movieBackImageBlobColumn, backImage) +} + +func (qb *movieQueryBuilder) destroyImages(ctx context.Context, movieID int) error { + if err := qb.DestroyImage(ctx, movieID, movieFrontImageBlobColumn); err != nil { return err } - return err + if err := qb.DestroyImage(ctx, movieID, movieBackImageBlobColumn); err != nil { + return err + } + + return nil } func (qb *movieQueryBuilder) GetFrontImage(ctx context.Context, movieID int) ([]byte, error) { - query := `SELECT front_image from movies_images WHERE movie_id = ?` - return getImage(ctx, qb.tx, query, movieID) + return qb.GetImage(ctx, movieID, movieFrontImageBlobColumn) } func (qb *movieQueryBuilder) GetBackImage(ctx context.Context, movieID int) ([]byte, error) { - query := `SELECT back_image from movies_images WHERE movie_id = ?` - return getImage(ctx, qb.tx, query, movieID) + return qb.GetImage(ctx, movieID, movieBackImageBlobColumn) } func (qb *movieQueryBuilder) FindByPerformerID(ctx context.Context, performerID int) ([]*models.Movie, error) { diff --git a/pkg/sqlite/movies_test.go b/pkg/sqlite/movies_test.go index eff0cf50b..9180dde20 100644 --- a/pkg/sqlite/movies_test.go +++ b/pkg/sqlite/movies_test.go @@ -15,12 +15,11 @@ import ( "github.com/stashapp/stash/pkg/hash/md5" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/sqlite" ) func TestMovieFindByName(t *testing.T) { withTxn(func(ctx context.Context) error { - mqb := sqlite.MovieReaderWriter + mqb := db.Movie name := movieNames[movieIdxWithScene] // find a movie by name @@ -53,7 +52,7 @@ func TestMovieFindByNames(t *testing.T) { withTxn(func(ctx context.Context) error { var names []string - mqb := sqlite.MovieReaderWriter + mqb := db.Movie names = append(names, movieNames[movieIdxWithScene]) // find movies by names @@ -76,9 +75,80 @@ func TestMovieFindByNames(t *testing.T) { }) } +func moviesToIDs(i []*models.Movie) []int { + ret := make([]int, len(i)) + for i, v := range i { + ret[i] = v.ID + } + + return ret +} + +func TestMovieQuery(t *testing.T) { + var ( + frontImage = "front_image" + backImage = "back_image" + ) + + tests := []struct { + name string + findFilter *models.FindFilterType + filter *models.MovieFilterType + includeIdxs []int + excludeIdxs []int + wantErr bool + }{ + { + "is missing front image", + nil, + &models.MovieFilterType{ + IsMissing: &frontImage, + }, + // just ensure that it doesn't error + nil, + nil, + false, + }, + { + "is missing back image", + nil, + &models.MovieFilterType{ + IsMissing: &backImage, + }, + // just ensure that it doesn't error + nil, + nil, + false, + }, + } + + for _, tt := range tests { + runWithRollbackTxn(t, tt.name, func(t *testing.T, ctx context.Context) { + assert := assert.New(t) + + results, _, err := db.Movie.Query(ctx, tt.filter, tt.findFilter) + if (err != nil) != tt.wantErr { + t.Errorf("MovieQueryBuilder.Query() error = %v, wantErr %v", err, tt.wantErr) + return + } + + ids := moviesToIDs(results) + include := indexesToIDs(performerIDs, tt.includeIdxs) + exclude := indexesToIDs(performerIDs, tt.excludeIdxs) + + for _, i := range include { + assert.Contains(ids, i) + } + for _, e := range exclude { + assert.NotContains(ids, e) + } + }) + } +} + func TestMovieQueryStudio(t *testing.T) { withTxn(func(ctx context.Context) error { - mqb := sqlite.MovieReaderWriter + mqb := db.Movie studioCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithMovie]), @@ -163,7 +233,7 @@ func TestMovieQueryURL(t *testing.T) { func verifyMovieQuery(t *testing.T, filter models.MovieFilterType, verifyFn func(s *models.Movie)) { withTxn(func(ctx context.Context) error { t.Helper() - sqb := sqlite.MovieReaderWriter + sqb := db.Movie movies := queryMovie(ctx, t, sqb, &filter, nil) @@ -196,7 +266,7 @@ func TestMovieQuerySorting(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.MovieReaderWriter + sqb := db.Movie movies := queryMovie(ctx, t, sqb, nil, &findFilter) // scenes should be in same order as indexes @@ -216,122 +286,50 @@ func TestMovieQuerySorting(t *testing.T) { }) } -func TestMovieUpdateMovieImages(t *testing.T) { - if err := withTxn(func(ctx context.Context) error { - mqb := sqlite.MovieReaderWriter +func TestMovieUpdateFrontImage(t *testing.T) { + if err := withRollbackTxn(func(ctx context.Context) error { + qb := db.Movie // create movie to test against const name = "TestMovieUpdateMovieImages" - movie := models.Movie{ + toCreate := models.Movie{ Name: sql.NullString{String: name, Valid: true}, Checksum: md5.FromString(name), } - created, err := mqb.Create(ctx, movie) + movie, err := qb.Create(ctx, toCreate) if err != nil { return fmt.Errorf("Error creating movie: %s", err.Error()) } - frontImage := []byte("frontImage") - backImage := []byte("backImage") - err = mqb.UpdateImages(ctx, created.ID, frontImage, backImage) - if err != nil { - return fmt.Errorf("Error updating movie images: %s", err.Error()) - } - - // ensure images are set - storedFront, err := mqb.GetFrontImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting front image: %s", err.Error()) - } - assert.Equal(t, storedFront, frontImage) - - storedBack, err := mqb.GetBackImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting back image: %s", err.Error()) - } - assert.Equal(t, storedBack, backImage) - - // set front image only - newImage := []byte("newImage") - err = mqb.UpdateImages(ctx, created.ID, newImage, nil) - if err != nil { - return fmt.Errorf("Error updating movie images: %s", err.Error()) - } - - storedFront, err = mqb.GetFrontImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting front image: %s", err.Error()) - } - assert.Equal(t, storedFront, newImage) - - // back image should be nil - storedBack, err = mqb.GetBackImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting back image: %s", err.Error()) - } - assert.Nil(t, nil) - - // set back image only - err = mqb.UpdateImages(ctx, created.ID, nil, newImage) - if err == nil { - return fmt.Errorf("Expected error setting nil front image") - } - - return nil + return testUpdateImage(t, ctx, movie.ID, qb.UpdateFrontImage, qb.GetFrontImage) }); err != nil { t.Error(err.Error()) } } -func TestMovieDestroyMovieImages(t *testing.T) { - if err := withTxn(func(ctx context.Context) error { - mqb := sqlite.MovieReaderWriter +func TestMovieUpdateBackImage(t *testing.T) { + if err := withRollbackTxn(func(ctx context.Context) error { + qb := db.Movie // create movie to test against - const name = "TestMovieDestroyMovieImages" - movie := models.Movie{ + const name = "TestMovieUpdateMovieImages" + toCreate := models.Movie{ Name: sql.NullString{String: name, Valid: true}, Checksum: md5.FromString(name), } - created, err := mqb.Create(ctx, movie) + movie, err := qb.Create(ctx, toCreate) if err != nil { return fmt.Errorf("Error creating movie: %s", err.Error()) } - frontImage := []byte("frontImage") - backImage := []byte("backImage") - err = mqb.UpdateImages(ctx, created.ID, frontImage, backImage) - if err != nil { - return fmt.Errorf("Error updating movie images: %s", err.Error()) - } - - err = mqb.DestroyImages(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error destroying movie images: %s", err.Error()) - } - - // front image should be nil - storedFront, err := mqb.GetFrontImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting front image: %s", err.Error()) - } - assert.Nil(t, storedFront) - - // back image should be nil - storedBack, err := mqb.GetBackImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting back image: %s", err.Error()) - } - assert.Nil(t, storedBack) - - return nil + return testUpdateImage(t, ctx, movie.ID, qb.UpdateBackImage, qb.GetBackImage) }); err != nil { t.Error(err.Error()) } } // TODO Update -// TODO Destroy +// TODO Destroy - ensure image is destroyed // TODO Find // TODO Count // TODO All diff --git a/pkg/sqlite/performer.go b/pkg/sqlite/performer.go index 024679007..f288401d3 100644 --- a/pkg/sqlite/performer.go +++ b/pkg/sqlite/performer.go @@ -23,7 +23,8 @@ const ( performersAliasesTable = "performer_aliases" performerAliasColumn = "alias" performersTagsTable = "performers_tags" - performersImageTable = "performers_image" // performer cover image + + performerImageBlobColumn = "image_blob" ) type performerRow struct { @@ -54,6 +55,9 @@ type performerRow struct { HairColor zero.String `db:"hair_color"` Weight null.Int `db:"weight"` IgnoreAutoTag bool `db:"ignore_auto_tag"` + + // not used for resolution + ImageBlob zero.String `db:"image_blob"` } func (r *performerRow) fromPerformer(o models.Performer) { @@ -159,16 +163,21 @@ func (r *performerRowRecord) fromPartial(o models.PerformerPartial) { type PerformerStore struct { repository + blobJoinQueryBuilder tableMgr *table } -func NewPerformerStore() *PerformerStore { +func NewPerformerStore(blobStore *BlobStore) *PerformerStore { return &PerformerStore{ repository: repository{ tableName: performerTable, idColumn: idColumn, }, + blobJoinQueryBuilder: blobJoinQueryBuilder{ + blobStore: blobStore, + joinTable: performerTable, + }, tableMgr: performerTableMgr, } } @@ -275,6 +284,11 @@ func (qb *PerformerStore) Update(ctx context.Context, updatedObject *models.Perf } func (qb *PerformerStore) Destroy(ctx context.Context, id int) error { + // must handle image checksums manually + if err := qb.DestroyImage(ctx, id); err != nil { + return err + } + return qb.destroyExisting(ctx, []int{id}) } @@ -690,8 +704,7 @@ func performerIsMissingCriterionHandler(qb *PerformerStore, isMissing *string) c f.addLeftJoin(performersScenesTable, "scenes_join", "scenes_join.performer_id = performers.id") f.addWhere("scenes_join.scene_id IS NULL") case "image": - f.addLeftJoin(performersImageTable, "image_join", "image_join.performer_id = performers.id") - f.addWhere("image_join.performer_id IS NULL") + f.addWhere("performers.image_blob IS NULL") case "stash_id": performersStashIDsTableMgr.join(f, "performer_stash_ids", "performers.id") f.addWhere("performer_stash_ids.performer_id IS NULL") @@ -911,27 +924,16 @@ func (qb *PerformerStore) GetTagIDs(ctx context.Context, id int) ([]int, error) return qb.tagsRepository().getIDs(ctx, id) } -func (qb *PerformerStore) imageRepository() *imageRepository { - return &imageRepository{ - repository: repository{ - tx: qb.tx, - tableName: "performers_image", - idColumn: performerIDColumn, - }, - imageColumn: "image", - } -} - func (qb *PerformerStore) GetImage(ctx context.Context, performerID int) ([]byte, error) { - return qb.imageRepository().get(ctx, performerID) + return qb.blobJoinQueryBuilder.GetImage(ctx, performerID, performerImageBlobColumn) } func (qb *PerformerStore) UpdateImage(ctx context.Context, performerID int, image []byte) error { - return qb.imageRepository().replace(ctx, performerID, image) + return qb.blobJoinQueryBuilder.UpdateImage(ctx, performerID, performerImageBlobColumn, image) } func (qb *PerformerStore) DestroyImage(ctx context.Context, performerID int) error { - return qb.imageRepository().destroy(ctx, []int{performerID}) + return qb.blobJoinQueryBuilder.DestroyImage(ctx, performerID, performerImageBlobColumn) } func (qb *PerformerStore) stashIDRepository() *stashIDRepository { diff --git a/pkg/sqlite/performer_test.go b/pkg/sqlite/performer_test.go index 0033e98a3..732c427a7 100644 --- a/pkg/sqlite/performer_test.go +++ b/pkg/sqlite/performer_test.go @@ -1029,26 +1029,7 @@ func TestPerformerUpdatePerformerImage(t *testing.T) { return fmt.Errorf("Error creating performer: %s", err.Error()) } - image := []byte("image") - err = qb.UpdateImage(ctx, performer.ID, image) - if err != nil { - return fmt.Errorf("Error updating performer image: %s", err.Error()) - } - - // ensure image set - storedImage, err := qb.GetImage(ctx, performer.ID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Equal(t, storedImage, image) - - // set nil image - err = qb.UpdateImage(ctx, performer.ID, nil) - if err == nil { - return fmt.Errorf("Expected error setting nil image") - } - - return nil + return testUpdateImage(t, ctx, performer.ID, qb.UpdateImage, qb.GetImage) }); err != nil { t.Error(err.Error()) } diff --git a/pkg/sqlite/repository.go b/pkg/sqlite/repository.go index fe7c6c7da..c3b1d74aa 100644 --- a/pkg/sqlite/repository.go +++ b/pkg/sqlite/repository.go @@ -387,29 +387,6 @@ func (r *joinRepository) replace(ctx context.Context, id int, foreignIDs []int) return nil } -type imageRepository struct { - repository - imageColumn string -} - -func (r *imageRepository) get(ctx context.Context, id int) ([]byte, error) { - query := fmt.Sprintf("SELECT %s from %s WHERE %s = ?", r.imageColumn, r.tableName, r.idColumn) - var ret []byte - err := r.querySimple(ctx, query, []interface{}{id}, &ret) - return ret, err -} - -func (r *imageRepository) replace(ctx context.Context, id int, image []byte) error { - if err := r.destroy(ctx, []int{id}); err != nil { - return err - } - - stmt := fmt.Sprintf("INSERT INTO %s (%s, %s) VALUES (?, ?)", r.tableName, r.idColumn, r.imageColumn) - _, err := r.tx.Exec(ctx, stmt, id, image) - - return err -} - type captionRepository struct { repository } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 58c652a2c..a5e903653 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -31,6 +31,8 @@ const ( scenesTagsTable = "scenes_tags" scenesGalleriesTable = "scenes_galleries" moviesScenesTable = "movies_scenes" + + sceneCoverBlobColumn = "cover_blob" ) var findExactDuplicateQuery = ` @@ -72,6 +74,9 @@ type sceneRow struct { ResumeTime float64 `db:"resume_time"` PlayDuration float64 `db:"play_duration"` PlayCount int `db:"play_count"` + + // not used in resolutions or updates + CoverBlob zero.String `db:"cover_blob"` } func (r *sceneRow) fromScene(o models.Scene) { @@ -172,6 +177,7 @@ func (r *sceneRowRecord) fromPartial(o models.ScenePartial) { type SceneStore struct { repository + blobJoinQueryBuilder tableMgr *table oCounterManager @@ -179,12 +185,16 @@ type SceneStore struct { fileStore *FileStore } -func NewSceneStore(fileStore *FileStore) *SceneStore { +func NewSceneStore(fileStore *FileStore, blobStore *BlobStore) *SceneStore { return &SceneStore{ repository: repository{ tableName: sceneTable, idColumn: idColumn, }, + blobJoinQueryBuilder: blobJoinQueryBuilder{ + blobStore: blobStore, + joinTable: sceneTable, + }, tableMgr: sceneTableMgr, oCounterManager: oCounterManager{sceneTableMgr}, @@ -353,6 +363,11 @@ func (qb *SceneStore) Update(ctx context.Context, updatedObject *models.Scene) e } func (qb *SceneStore) Destroy(ctx context.Context, id int) error { + // must handle image checksums manually + if err := qb.destroyCover(ctx, id); err != nil { + return err + } + // scene markers should be handled prior to calling destroy // galleries should be handled prior to calling destroy @@ -1187,6 +1202,8 @@ func sceneIsMissingCriterionHandler(qb *SceneStore, isMissing *string) criterion qb.addSceneFilesTable(f) f.addLeftJoin(fingerprintTable, "fingerprints_phash", "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'") f.addWhere("fingerprints_phash.fingerprint IS NULL") + case "cover": + f.addWhere("scenes.cover_blob IS NULL") default: f.addWhere("(scenes." + *isMissing + " IS NULL OR TRIM(scenes." + *isMissing + ") = '')") } @@ -1464,17 +1481,6 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF } } -func (qb *SceneStore) imageRepository() *imageRepository { - return &imageRepository{ - repository: repository{ - tx: qb.tx, - tableName: "scenes_cover", - idColumn: sceneIDColumn, - }, - imageColumn: "cover", - } -} - func (qb *SceneStore) getPlayCount(ctx context.Context, id int) (int, error) { q := dialect.From(qb.tableMgr.table).Select("play_count").Where(goqu.Ex{"id": id}) @@ -1532,15 +1538,19 @@ func (qb *SceneStore) IncrementWatchCount(ctx context.Context, id int) (int, err } func (qb *SceneStore) GetCover(ctx context.Context, sceneID int) ([]byte, error) { - return qb.imageRepository().get(ctx, sceneID) + return qb.GetImage(ctx, sceneID, sceneCoverBlobColumn) +} + +func (qb *SceneStore) HasCover(ctx context.Context, sceneID int) (bool, error) { + return qb.HasImage(ctx, sceneID, sceneCoverBlobColumn) } func (qb *SceneStore) UpdateCover(ctx context.Context, sceneID int, image []byte) error { - return qb.imageRepository().replace(ctx, sceneID, image) + return qb.UpdateImage(ctx, sceneID, sceneCoverBlobColumn, image) } -func (qb *SceneStore) DestroyCover(ctx context.Context, sceneID int) error { - return qb.imageRepository().destroy(ctx, []int{sceneID}) +func (qb *SceneStore) destroyCover(ctx context.Context, sceneID int) error { + return qb.DestroyImage(ctx, sceneID, sceneCoverBlobColumn) } func (qb *SceneStore) AssignFiles(ctx context.Context, sceneID int, fileIDs []file.ID) error { diff --git a/pkg/sqlite/scene_test.go b/pkg/sqlite/scene_test.go index 697dba113..e4fc7a3f3 100644 --- a/pkg/sqlite/scene_test.go +++ b/pkg/sqlite/scene_test.go @@ -4088,53 +4088,7 @@ func TestSceneUpdateSceneCover(t *testing.T) { sceneID := sceneIDs[sceneIdxWithGallery] - image := []byte("image") - if err := qb.UpdateCover(ctx, sceneID, image); err != nil { - return fmt.Errorf("Error updating scene cover: %s", err.Error()) - } - - // ensure image set - storedImage, err := qb.GetCover(ctx, sceneID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Equal(t, storedImage, image) - - // set nil image - err = qb.UpdateCover(ctx, sceneID, nil) - if err == nil { - return fmt.Errorf("Expected error setting nil image") - } - - return nil - }); err != nil { - t.Error(err.Error()) - } -} - -func TestSceneDestroySceneCover(t *testing.T) { - if err := withTxn(func(ctx context.Context) error { - qb := db.Scene - - sceneID := sceneIDs[sceneIdxWithGallery] - - image := []byte("image") - if err := qb.UpdateCover(ctx, sceneID, image); err != nil { - return fmt.Errorf("Error updating scene image: %s", err.Error()) - } - - if err := qb.DestroyCover(ctx, sceneID); err != nil { - return fmt.Errorf("Error destroying scene cover: %s", err.Error()) - } - - // image should be nil - storedImage, err := qb.GetCover(ctx, sceneID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Nil(t, storedImage) - - return nil + return testUpdateImage(t, ctx, sceneID, qb.UpdateCover, qb.GetCover) }); err != nil { t.Error(err.Error()) } diff --git a/pkg/sqlite/setup_test.go b/pkg/sqlite/setup_test.go index 2d034dd50..4300111cf 100644 --- a/pkg/sqlite/setup_test.go +++ b/pkg/sqlite/setup_test.go @@ -537,6 +537,10 @@ func runTests(m *testing.M) int { f.Close() databaseFile := f.Name() db = sqlite.NewDatabase() + db.SetBlobStoreOptions(sqlite.BlobStoreOptions{ + UseDatabase: true, + // don't use filesystem + }) if err := db.Open(databaseFile); err != nil { panic(fmt.Sprintf("Could not initialize database: %s", err.Error())) @@ -566,11 +570,11 @@ func populateDB() error { // TODO - link folders to zip files - if err := createMovies(ctx, sqlite.MovieReaderWriter, moviesNameCase, moviesNameNoCase); err != nil { + if err := createMovies(ctx, db.Movie, moviesNameCase, moviesNameNoCase); err != nil { return fmt.Errorf("error creating movies: %s", err.Error()) } - if err := createTags(ctx, sqlite.TagReaderWriter, tagsNameCase, tagsNameNoCase); err != nil { + if err := createTags(ctx, db.Tag, tagsNameCase, tagsNameNoCase); err != nil { return fmt.Errorf("error creating tags: %s", err.Error()) } @@ -578,7 +582,7 @@ func populateDB() error { return fmt.Errorf("error creating performers: %s", err.Error()) } - if err := createStudios(ctx, sqlite.StudioReaderWriter, studiosNameCase, studiosNameNoCase); err != nil { + if err := createStudios(ctx, db.Studio, studiosNameCase, studiosNameNoCase); err != nil { return fmt.Errorf("error creating studios: %s", err.Error()) } @@ -594,7 +598,7 @@ func populateDB() error { return fmt.Errorf("error creating images: %s", err.Error()) } - if err := addTagImage(ctx, sqlite.TagReaderWriter, tagIdxWithCoverImage); err != nil { + if err := addTagImage(ctx, db.Tag, tagIdxWithCoverImage); err != nil { return fmt.Errorf("error adding tag image: %s", err.Error()) } @@ -602,15 +606,15 @@ func populateDB() error { return fmt.Errorf("error creating saved filters: %s", err.Error()) } - if err := linkMovieStudios(ctx, sqlite.MovieReaderWriter); err != nil { + if err := linkMovieStudios(ctx, db.Movie); err != nil { return fmt.Errorf("error linking movie studios: %s", err.Error()) } - if err := linkStudiosParent(ctx, sqlite.StudioReaderWriter); err != nil { + if err := linkStudiosParent(ctx, db.Studio); err != nil { return fmt.Errorf("error linking studios parent: %s", err.Error()) } - if err := linkTagsParent(ctx, sqlite.TagReaderWriter); err != nil { + if err := linkTagsParent(ctx, db.Tag); err != nil { return fmt.Errorf("error linking tags parent: %s", err.Error()) } diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index 95b7d2f10..af864df01 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -1,9 +1,6 @@ package sqlite import ( - "context" - "database/sql" - "errors" "fmt" "math/rand" "regexp" @@ -290,28 +287,6 @@ func getCountCriterionClause(primaryTable, joinTable, primaryFK string, criterio return getIntCriterionWhereClause(lhs, criterion) } -func getImage(ctx context.Context, tx dbWrapper, query string, args ...interface{}) ([]byte, error) { - rows, err := tx.Queryx(ctx, query, args...) - - if err != nil && !errors.Is(err, sql.ErrNoRows) { - return nil, err - } - defer rows.Close() - - var ret []byte - if rows.Next() { - if err := rows.Scan(&ret); err != nil { - return nil, err - } - } - - if err := rows.Err(); err != nil { - return nil, err - } - - return ret, nil -} - func coalesce(column string) string { return fmt.Sprintf("COALESCE(%s, '')", column) } diff --git a/pkg/sqlite/studio.go b/pkg/sqlite/studio.go index f3dd963a8..04995a1db 100644 --- a/pkg/sqlite/studio.go +++ b/pkg/sqlite/studio.go @@ -13,20 +13,31 @@ import ( "github.com/stashapp/stash/pkg/sliceutil/intslice" ) -const studioTable = "studios" -const studioIDColumn = "studio_id" -const studioAliasesTable = "studio_aliases" -const studioAliasColumn = "alias" +const ( + studioTable = "studios" + studioIDColumn = "studio_id" + studioAliasesTable = "studio_aliases" + studioAliasColumn = "alias" + + studioImageBlobColumn = "image_blob" +) type studioQueryBuilder struct { repository + blobJoinQueryBuilder } -var StudioReaderWriter = &studioQueryBuilder{ - repository{ - tableName: studioTable, - idColumn: idColumn, - }, +func NewStudioReaderWriter(blobStore *BlobStore) *studioQueryBuilder { + return &studioQueryBuilder{ + repository{ + tableName: studioTable, + idColumn: idColumn, + }, + blobJoinQueryBuilder{ + blobStore: blobStore, + joinTable: studioTable, + }, + } } func (qb *studioQueryBuilder) Create(ctx context.Context, newObject models.Studio) (*models.Studio, error) { @@ -57,6 +68,11 @@ func (qb *studioQueryBuilder) UpdateFull(ctx context.Context, updatedObject mode } func (qb *studioQueryBuilder) Destroy(ctx context.Context, id int) error { + // must handle image checksums manually + if err := qb.destroyImage(ctx, id); err != nil { + return err + } + // TODO - set null on foreign key in scraped items // remove studio from scraped items _, err := qb.tx.Exec(ctx, "UPDATE scraped_items SET studio_id = null WHERE studio_id = ?", id) @@ -428,31 +444,20 @@ func (qb *studioQueryBuilder) queryStudios(ctx context.Context, query string, ar return []*models.Studio(ret), nil } -func (qb *studioQueryBuilder) imageRepository() *imageRepository { - return &imageRepository{ - repository: repository{ - tx: qb.tx, - tableName: "studios_image", - idColumn: studioIDColumn, - }, - imageColumn: "image", - } -} - func (qb *studioQueryBuilder) GetImage(ctx context.Context, studioID int) ([]byte, error) { - return qb.imageRepository().get(ctx, studioID) + return qb.blobJoinQueryBuilder.GetImage(ctx, studioID, studioImageBlobColumn) } func (qb *studioQueryBuilder) HasImage(ctx context.Context, studioID int) (bool, error) { - return qb.imageRepository().exists(ctx, studioID) + return qb.blobJoinQueryBuilder.HasImage(ctx, studioID, studioImageBlobColumn) } func (qb *studioQueryBuilder) UpdateImage(ctx context.Context, studioID int, image []byte) error { - return qb.imageRepository().replace(ctx, studioID, image) + return qb.blobJoinQueryBuilder.UpdateImage(ctx, studioID, studioImageBlobColumn, image) } -func (qb *studioQueryBuilder) DestroyImage(ctx context.Context, studioID int) error { - return qb.imageRepository().destroy(ctx, []int{studioID}) +func (qb *studioQueryBuilder) destroyImage(ctx context.Context, studioID int) error { + return qb.blobJoinQueryBuilder.DestroyImage(ctx, studioID, studioImageBlobColumn) } func (qb *studioQueryBuilder) stashIDRepository() *stashIDRepository { diff --git a/pkg/sqlite/studio_test.go b/pkg/sqlite/studio_test.go index 4bffe4517..334ad1a15 100644 --- a/pkg/sqlite/studio_test.go +++ b/pkg/sqlite/studio_test.go @@ -14,13 +14,12 @@ import ( "testing" "github.com/stashapp/stash/pkg/models" - "github.com/stashapp/stash/pkg/sqlite" "github.com/stretchr/testify/assert" ) func TestStudioFindByName(t *testing.T) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio name := studioNames[studioIdxWithScene] // find a studio by name @@ -70,7 +69,7 @@ func TestStudioQueryNameOr(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios := queryStudio(ctx, t, sqb, &studioFilter, nil) @@ -101,7 +100,7 @@ func TestStudioQueryNameAndUrl(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios := queryStudio(ctx, t, sqb, &studioFilter, nil) @@ -136,7 +135,7 @@ func TestStudioQueryNameNotUrl(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios := queryStudio(ctx, t, sqb, &studioFilter, nil) @@ -167,7 +166,7 @@ func TestStudioIllegalQuery(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio _, _, err := sqb.Query(ctx, studioFilter, nil) assert.NotNil(err) @@ -193,7 +192,7 @@ func TestStudioQueryIgnoreAutoTag(t *testing.T) { IgnoreAutoTag: &ignoreAutoTag, } - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios := queryStudio(ctx, t, sqb, &studioFilter, nil) @@ -208,7 +207,7 @@ func TestStudioQueryIgnoreAutoTag(t *testing.T) { func TestStudioQueryForAutoTag(t *testing.T) { withTxn(func(ctx context.Context) error { - tqb := sqlite.StudioReaderWriter + tqb := db.Studio name := studioNames[studioIdxWithMovie] // find a studio by name @@ -239,7 +238,7 @@ func TestStudioQueryForAutoTag(t *testing.T) { func TestStudioQueryParent(t *testing.T) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studioCriterion := models.MultiCriterionInput{ Value: []string{ strconv.Itoa(studioIDs[studioIdxWithChildStudio]), @@ -289,18 +288,18 @@ func TestStudioDestroyParent(t *testing.T) { // create parent and child studios if err := withTxn(func(ctx context.Context) error { - createdParent, err := createStudio(ctx, sqlite.StudioReaderWriter, parentName, nil) + createdParent, err := createStudio(ctx, db.Studio, parentName, nil) if err != nil { return fmt.Errorf("Error creating parent studio: %s", err.Error()) } parentID := int64(createdParent.ID) - createdChild, err := createStudio(ctx, sqlite.StudioReaderWriter, childName, &parentID) + createdChild, err := createStudio(ctx, db.Studio, childName, &parentID) if err != nil { return fmt.Errorf("Error creating child studio: %s", err.Error()) } - sqb := sqlite.StudioReaderWriter + sqb := db.Studio // destroy the parent err = sqb.Destroy(ctx, createdParent.ID) @@ -322,7 +321,7 @@ func TestStudioDestroyParent(t *testing.T) { func TestStudioFindChildren(t *testing.T) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios, err := sqb.FindChildren(ctx, studioIDs[studioIdxWithChildStudio]) @@ -351,18 +350,18 @@ func TestStudioUpdateClearParent(t *testing.T) { // create parent and child studios if err := withTxn(func(ctx context.Context) error { - createdParent, err := createStudio(ctx, sqlite.StudioReaderWriter, parentName, nil) + createdParent, err := createStudio(ctx, db.Studio, parentName, nil) if err != nil { return fmt.Errorf("Error creating parent studio: %s", err.Error()) } parentID := int64(createdParent.ID) - createdChild, err := createStudio(ctx, sqlite.StudioReaderWriter, childName, &parentID) + createdChild, err := createStudio(ctx, db.Studio, childName, &parentID) if err != nil { return fmt.Errorf("Error creating child studio: %s", err.Error()) } - sqb := sqlite.StudioReaderWriter + sqb := db.Studio // clear the parent id from the child updatePartial := models.StudioPartial{ @@ -388,70 +387,16 @@ func TestStudioUpdateClearParent(t *testing.T) { func TestStudioUpdateStudioImage(t *testing.T) { if err := withTxn(func(ctx context.Context) error { - qb := sqlite.StudioReaderWriter + qb := db.Studio - // create performer to test against + // create studio to test against const name = "TestStudioUpdateStudioImage" - created, err := createStudio(ctx, sqlite.StudioReaderWriter, name, nil) + created, err := createStudio(ctx, db.Studio, name, nil) if err != nil { return fmt.Errorf("Error creating studio: %s", err.Error()) } - image := []byte("image") - err = qb.UpdateImage(ctx, created.ID, image) - if err != nil { - return fmt.Errorf("Error updating studio image: %s", err.Error()) - } - - // ensure image set - storedImage, err := qb.GetImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Equal(t, storedImage, image) - - // set nil image - err = qb.UpdateImage(ctx, created.ID, nil) - if err == nil { - return fmt.Errorf("Expected error setting nil image") - } - - return nil - }); err != nil { - t.Error(err.Error()) - } -} - -func TestStudioDestroyStudioImage(t *testing.T) { - if err := withTxn(func(ctx context.Context) error { - qb := sqlite.StudioReaderWriter - - // create performer to test against - const name = "TestStudioDestroyStudioImage" - created, err := createStudio(ctx, sqlite.StudioReaderWriter, name, nil) - if err != nil { - return fmt.Errorf("Error creating studio: %s", err.Error()) - } - - image := []byte("image") - err = qb.UpdateImage(ctx, created.ID, image) - if err != nil { - return fmt.Errorf("Error updating studio image: %s", err.Error()) - } - - err = qb.DestroyImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error destroying studio image: %s", err.Error()) - } - - // image should be nil - storedImage, err := qb.GetImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Nil(t, storedImage) - - return nil + return testUpdateImage(t, ctx, created.ID, qb.UpdateImage, qb.GetImage) }); err != nil { t.Error(err.Error()) } @@ -478,7 +423,7 @@ func TestStudioQuerySceneCount(t *testing.T) { func verifyStudiosSceneCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studioFilter := models.StudioFilterType{ SceneCount: &sceneCountCriterion, } @@ -519,7 +464,7 @@ func TestStudioQueryImageCount(t *testing.T) { func verifyStudiosImageCount(t *testing.T, imageCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studioFilter := models.StudioFilterType{ ImageCount: &imageCountCriterion, } @@ -575,7 +520,7 @@ func TestStudioQueryGalleryCount(t *testing.T) { func verifyStudiosGalleryCount(t *testing.T, galleryCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studioFilter := models.StudioFilterType{ GalleryCount: &galleryCountCriterion, } @@ -606,11 +551,11 @@ func verifyStudiosGalleryCount(t *testing.T, galleryCountCriterion models.IntCri func TestStudioStashIDs(t *testing.T) { if err := withTxn(func(ctx context.Context) error { - qb := sqlite.StudioReaderWriter + qb := db.Studio // create studio to test against const name = "TestStudioStashIDs" - created, err := createStudio(ctx, sqlite.StudioReaderWriter, name, nil) + created, err := createStudio(ctx, db.Studio, name, nil) if err != nil { return fmt.Errorf("Error creating studio: %s", err.Error()) } @@ -688,7 +633,7 @@ func TestStudioQueryRating(t *testing.T) { func verifyStudioQuery(t *testing.T, filter models.StudioFilterType, verifyFn func(ctx context.Context, s *models.Studio)) { withTxn(func(ctx context.Context) error { t.Helper() - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studios := queryStudio(ctx, t, sqb, &filter, nil) @@ -705,7 +650,7 @@ func verifyStudioQuery(t *testing.T, filter models.StudioFilterType, verifyFn fu func verifyStudiosRating(t *testing.T, ratingCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio studioFilter := models.StudioFilterType{ Rating: &ratingCriterion, } @@ -726,7 +671,7 @@ func verifyStudiosRating(t *testing.T, ratingCriterion models.IntCriterionInput) func TestStudioQueryIsMissingRating(t *testing.T) { withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio isMissing := "rating" studioFilter := models.StudioFilterType{ IsMissing: &isMissing, @@ -802,7 +747,7 @@ func TestStudioQueryAlias(t *testing.T) { verifyFn := func(ctx context.Context, studio *models.Studio) { t.Helper() - aliases, err := sqlite.StudioReaderWriter.GetAliases(ctx, studio.ID) + aliases, err := db.Studio.GetAliases(ctx, studio.ID) if err != nil { t.Errorf("Error querying studios: %s", err.Error()) } @@ -837,7 +782,7 @@ func TestStudioQueryAlias(t *testing.T) { func TestStudioUpdateAlias(t *testing.T) { if err := withTxn(func(ctx context.Context) error { - qb := sqlite.StudioReaderWriter + qb := db.Studio // create studio to test against const name = "TestStudioUpdateAlias" @@ -934,7 +879,7 @@ func TestStudioQueryFast(t *testing.T) { } withTxn(func(ctx context.Context) error { - sqb := sqlite.StudioReaderWriter + sqb := db.Studio for _, f := range filters { for _, ff := range findFilters { _, _, err := sqb.Query(ctx, &f, &ff) diff --git a/pkg/sqlite/tables.go b/pkg/sqlite/tables.go index 3cf0f5bce..2bf1bfd16 100644 --- a/pkg/sqlite/tables.go +++ b/pkg/sqlite/tables.go @@ -234,3 +234,10 @@ var ( idColumn: goqu.T(movieTable).Col(idColumn), } ) + +var ( + blobTableMgr = &table{ + table: goqu.T(blobTable), + idColumn: goqu.T(blobTable).Col(blobChecksumColumn), + } +) diff --git a/pkg/sqlite/tag.go b/pkg/sqlite/tag.go index aa6232cb2..9ad4abcaf 100644 --- a/pkg/sqlite/tag.go +++ b/pkg/sqlite/tag.go @@ -13,20 +13,31 @@ import ( "github.com/stashapp/stash/pkg/sliceutil/intslice" ) -const tagTable = "tags" -const tagIDColumn = "tag_id" -const tagAliasesTable = "tag_aliases" -const tagAliasColumn = "alias" +const ( + tagTable = "tags" + tagIDColumn = "tag_id" + tagAliasesTable = "tag_aliases" + tagAliasColumn = "alias" + + tagImageBlobColumn = "image_blob" +) type tagQueryBuilder struct { repository + blobJoinQueryBuilder } -var TagReaderWriter = &tagQueryBuilder{ - repository{ - tableName: tagTable, - idColumn: idColumn, - }, +func NewTagReaderWriter(blobStore *BlobStore) *tagQueryBuilder { + return &tagQueryBuilder{ + repository{ + tableName: tagTable, + idColumn: idColumn, + }, + blobJoinQueryBuilder{ + blobStore: blobStore, + joinTable: tagTable, + }, + } } func (qb *tagQueryBuilder) Create(ctx context.Context, newObject models.Tag) (*models.Tag, error) { @@ -57,16 +68,8 @@ func (qb *tagQueryBuilder) UpdateFull(ctx context.Context, updatedObject models. } func (qb *tagQueryBuilder) Destroy(ctx context.Context, id int) error { - // TODO - add delete cascade to foreign key - // delete tag from scenes and markers first - _, err := qb.tx.Exec(ctx, "DELETE FROM scenes_tags WHERE tag_id = ?", id) - if err != nil { - return err - } - - // TODO - add delete cascade to foreign key - _, err = qb.tx.Exec(ctx, "DELETE FROM scene_markers_tags WHERE tag_id = ?", id) - if err != nil { + // must handle image checksums manually + if err := qb.destroyImage(ctx, id); err != nil { return err } @@ -407,8 +410,7 @@ func tagIsMissingCriterionHandler(qb *tagQueryBuilder, isMissing *string) criter if isMissing != nil && *isMissing != "" { switch *isMissing { case "image": - qb.imageRepository().join(f, "", "tags.id") - f.addWhere("tags_image.tag_id IS NULL") + f.addWhere("tags.image_blob IS NULL") default: f.addWhere("(tags." + *isMissing + " IS NULL OR TRIM(tags." + *isMissing + ") = '')") } @@ -642,31 +644,16 @@ func (qb *tagQueryBuilder) queryTags(ctx context.Context, query string, args []i return []*models.Tag(ret), nil } -func (qb *tagQueryBuilder) imageRepository() *imageRepository { - return &imageRepository{ - repository: repository{ - tx: qb.tx, - tableName: "tags_image", - idColumn: tagIDColumn, - }, - imageColumn: "image", - } -} - func (qb *tagQueryBuilder) GetImage(ctx context.Context, tagID int) ([]byte, error) { - return qb.imageRepository().get(ctx, tagID) -} - -func (qb *tagQueryBuilder) HasImage(ctx context.Context, tagID int) (bool, error) { - return qb.imageRepository().exists(ctx, tagID) + return qb.blobJoinQueryBuilder.GetImage(ctx, tagID, tagImageBlobColumn) } func (qb *tagQueryBuilder) UpdateImage(ctx context.Context, tagID int, image []byte) error { - return qb.imageRepository().replace(ctx, tagID, image) + return qb.blobJoinQueryBuilder.UpdateImage(ctx, tagID, tagImageBlobColumn, image) } -func (qb *tagQueryBuilder) DestroyImage(ctx context.Context, tagID int) error { - return qb.imageRepository().destroy(ctx, []int{tagID}) +func (qb *tagQueryBuilder) destroyImage(ctx context.Context, tagID int) error { + return qb.blobJoinQueryBuilder.DestroyImage(ctx, tagID, tagImageBlobColumn) } func (qb *tagQueryBuilder) aliasRepository() *stringRepository { diff --git a/pkg/sqlite/tag_test.go b/pkg/sqlite/tag_test.go index a351f28f4..d3ff5459f 100644 --- a/pkg/sqlite/tag_test.go +++ b/pkg/sqlite/tag_test.go @@ -19,7 +19,7 @@ import ( func TestMarkerFindBySceneMarkerID(t *testing.T) { withTxn(func(ctx context.Context) error { - tqb := sqlite.TagReaderWriter + tqb := db.Tag markerID := markerIDs[markerIdxWithTag] @@ -46,7 +46,7 @@ func TestMarkerFindBySceneMarkerID(t *testing.T) { func TestTagFindByName(t *testing.T) { withTxn(func(ctx context.Context) error { - tqb := sqlite.TagReaderWriter + tqb := db.Tag name := tagNames[tagIdxWithScene] // find a tag by name @@ -82,7 +82,7 @@ func TestTagQueryIgnoreAutoTag(t *testing.T) { IgnoreAutoTag: &ignoreAutoTag, } - sqb := sqlite.TagReaderWriter + sqb := db.Tag tags := queryTags(ctx, t, sqb, &tagFilter, nil) @@ -97,7 +97,7 @@ func TestTagQueryIgnoreAutoTag(t *testing.T) { func TestTagQueryForAutoTag(t *testing.T) { withTxn(func(ctx context.Context) error { - tqb := sqlite.TagReaderWriter + tqb := db.Tag name := tagNames[tagIdx1WithScene] // find a tag by name @@ -131,7 +131,7 @@ func TestTagFindByNames(t *testing.T) { var names []string withTxn(func(ctx context.Context) error { - tqb := sqlite.TagReaderWriter + tqb := db.Tag names = append(names, tagNames[tagIdxWithScene]) // find tags by names @@ -176,7 +176,7 @@ func TestTagFindByNames(t *testing.T) { func TestTagQuerySort(t *testing.T) { withTxn(func(ctx context.Context) error { - sqb := sqlite.TagReaderWriter + sqb := db.Tag sortBy := "scenes_count" dir := models.SortDirectionEnumDesc @@ -253,7 +253,7 @@ func TestTagQueryAlias(t *testing.T) { } verifyFn := func(ctx context.Context, tag *models.Tag) { - aliases, err := sqlite.TagReaderWriter.GetAliases(ctx, tag.ID) + aliases, err := db.Tag.GetAliases(ctx, tag.ID) if err != nil { t.Errorf("Error querying tags: %s", err.Error()) } @@ -288,7 +288,7 @@ func TestTagQueryAlias(t *testing.T) { func verifyTagQuery(t *testing.T, tagFilter *models.TagFilterType, findFilter *models.FindFilterType, verifyFn func(ctx context.Context, t *models.Tag)) { withTxn(func(ctx context.Context) error { - sqb := sqlite.TagReaderWriter + sqb := db.Tag tags := queryTags(ctx, t, sqb, tagFilter, findFilter) @@ -312,7 +312,7 @@ func queryTags(ctx context.Context, t *testing.T, qb models.TagReader, tagFilter func TestTagQueryIsMissingImage(t *testing.T) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag isMissing := "image" tagFilter := models.TagFilterType{ IsMissing: &isMissing, @@ -366,7 +366,7 @@ func TestTagQuerySceneCount(t *testing.T) { func verifyTagSceneCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ SceneCount: &sceneCountCriterion, } @@ -408,7 +408,7 @@ func TestTagQueryMarkerCount(t *testing.T) { func verifyTagMarkerCount(t *testing.T, markerCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ MarkerCount: &markerCountCriterion, } @@ -450,7 +450,7 @@ func TestTagQueryImageCount(t *testing.T) { func verifyTagImageCount(t *testing.T, imageCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ ImageCount: &imageCountCriterion, } @@ -492,7 +492,7 @@ func TestTagQueryGalleryCount(t *testing.T) { func verifyTagGalleryCount(t *testing.T, imageCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ GalleryCount: &imageCountCriterion, } @@ -534,7 +534,7 @@ func TestTagQueryPerformerCount(t *testing.T) { func verifyTagPerformerCount(t *testing.T, imageCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ PerformerCount: &imageCountCriterion, } @@ -576,7 +576,7 @@ func TestTagQueryParentCount(t *testing.T) { func verifyTagParentCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ ParentCount: &sceneCountCriterion, } @@ -619,7 +619,7 @@ func TestTagQueryChildCount(t *testing.T) { func verifyTagChildCount(t *testing.T, sceneCountCriterion models.IntCriterionInput) { withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag tagFilter := models.TagFilterType{ ChildCount: &sceneCountCriterion, } @@ -644,7 +644,7 @@ func verifyTagChildCount(t *testing.T, sceneCountCriterion models.IntCriterionIn func TestTagQueryParent(t *testing.T) { withTxn(func(ctx context.Context) error { const nameField = "Name" - sqb := sqlite.TagReaderWriter + sqb := db.Tag tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithChildTag]), @@ -722,7 +722,7 @@ func TestTagQueryChild(t *testing.T) { withTxn(func(ctx context.Context) error { const nameField = "Name" - sqb := sqlite.TagReaderWriter + sqb := db.Tag tagCriterion := models.HierarchicalMultiCriterionInput{ Value: []string{ strconv.Itoa(tagIDs[tagIdxWithParentTag]), @@ -798,7 +798,7 @@ func TestTagQueryChild(t *testing.T) { func TestTagUpdateTagImage(t *testing.T) { if err := withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag // create tag to test against const name = "TestTagUpdateTagImage" @@ -810,64 +810,7 @@ func TestTagUpdateTagImage(t *testing.T) { return fmt.Errorf("Error creating tag: %s", err.Error()) } - image := []byte("image") - err = qb.UpdateImage(ctx, created.ID, image) - if err != nil { - return fmt.Errorf("Error updating studio image: %s", err.Error()) - } - - // ensure image set - storedImage, err := qb.GetImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Equal(t, storedImage, image) - - // set nil image - err = qb.UpdateImage(ctx, created.ID, nil) - if err == nil { - return fmt.Errorf("Expected error setting nil image") - } - - return nil - }); err != nil { - t.Error(err.Error()) - } -} - -func TestTagDestroyTagImage(t *testing.T) { - if err := withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter - - // create performer to test against - const name = "TestTagDestroyTagImage" - tag := models.Tag{ - Name: name, - } - created, err := qb.Create(ctx, tag) - if err != nil { - return fmt.Errorf("Error creating tag: %s", err.Error()) - } - - image := []byte("image") - err = qb.UpdateImage(ctx, created.ID, image) - if err != nil { - return fmt.Errorf("Error updating studio image: %s", err.Error()) - } - - err = qb.DestroyImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error destroying studio image: %s", err.Error()) - } - - // image should be nil - storedImage, err := qb.GetImage(ctx, created.ID) - if err != nil { - return fmt.Errorf("Error getting image: %s", err.Error()) - } - assert.Nil(t, storedImage) - - return nil + return testUpdateImage(t, ctx, created.ID, qb.UpdateImage, qb.GetImage) }); err != nil { t.Error(err.Error()) } @@ -875,7 +818,7 @@ func TestTagDestroyTagImage(t *testing.T) { func TestTagUpdateAlias(t *testing.T) { if err := withTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag // create tag to test against const name = "TestTagUpdateAlias" @@ -911,7 +854,7 @@ func TestTagMerge(t *testing.T) { // merge tests - perform these in a transaction that we'll rollback if err := withRollbackTxn(func(ctx context.Context) error { - qb := sqlite.TagReaderWriter + qb := db.Tag // try merging into same tag err := qb.Merge(ctx, []int{tagIDs[tagIdx1WithScene]}, tagIDs[tagIdx1WithScene]) diff --git a/pkg/sqlite/transaction.go b/pkg/sqlite/transaction.go index f67a2d6e7..743ccce04 100644 --- a/pkg/sqlite/transaction.go +++ b/pkg/sqlite/transaction.go @@ -131,13 +131,13 @@ func (db *Database) TxnRepository() models.Repository { Gallery: db.Gallery, GalleryChapter: GalleryChapterReaderWriter, Image: db.Image, - Movie: MovieReaderWriter, + Movie: db.Movie, Performer: db.Performer, Scene: db.Scene, SceneMarker: SceneMarkerReaderWriter, ScrapedItem: ScrapedItemReaderWriter, - Studio: StudioReaderWriter, - Tag: TagReaderWriter, + Studio: db.Studio, + Tag: db.Tag, SavedFilter: SavedFilterReaderWriter, } } diff --git a/pkg/txn/hooks.go b/pkg/txn/hooks.go index 5f36d7def..2327349cb 100644 --- a/pkg/txn/hooks.go +++ b/pkg/txn/hooks.go @@ -11,9 +11,10 @@ const ( ) type hookManager struct { - postCommitHooks []TxnFunc - postRollbackHooks []TxnFunc - postCompleteHooks []TxnFunc + preCommitHooks []TxnFunc + postCommitHooks []MustFunc + postRollbackHooks []MustFunc + postCompleteHooks []MustFunc } func (m *hookManager) register(ctx context.Context) context.Context { @@ -28,39 +29,55 @@ func hookManagerCtx(ctx context.Context) *hookManager { return m } -func executeHooks(ctx context.Context, hooks []TxnFunc) { +func executeHooks(ctx context.Context, hooks []TxnFunc) error { + // we need to return the first error for _, h := range hooks { - // ignore errors - _ = h(ctx) + if err := h(ctx); err != nil { + return err + } + } + + return nil +} + +func executeMustHooks(ctx context.Context, hooks []MustFunc) { + for _, h := range hooks { + h(ctx) } } -func executePostCommitHooks(ctx context.Context, outerCtx context.Context) { - m := hookManagerCtx(ctx) - executeHooks(outerCtx, m.postCommitHooks) +func (m *hookManager) executePostCommitHooks(ctx context.Context) { + executeMustHooks(ctx, m.postCommitHooks) } -func executePostRollbackHooks(ctx context.Context, outerCtx context.Context) { - m := hookManagerCtx(ctx) - executeHooks(outerCtx, m.postRollbackHooks) +func (m *hookManager) executePostRollbackHooks(ctx context.Context) { + executeMustHooks(ctx, m.postRollbackHooks) } -func executePostCompleteHooks(ctx context.Context, outerCtx context.Context) { - m := hookManagerCtx(ctx) - executeHooks(outerCtx, m.postCompleteHooks) +func (m *hookManager) executePreCommitHooks(ctx context.Context) error { + return executeHooks(ctx, m.preCommitHooks) } -func AddPostCommitHook(ctx context.Context, hook TxnFunc) { +func (m *hookManager) executePostCompleteHooks(ctx context.Context) { + executeMustHooks(ctx, m.postCompleteHooks) +} + +func AddPreCommitHook(ctx context.Context, hook TxnFunc) { + m := hookManagerCtx(ctx) + m.preCommitHooks = append(m.preCommitHooks, hook) +} + +func AddPostCommitHook(ctx context.Context, hook MustFunc) { m := hookManagerCtx(ctx) m.postCommitHooks = append(m.postCommitHooks, hook) } -func AddPostRollbackHook(ctx context.Context, hook TxnFunc) { +func AddPostRollbackHook(ctx context.Context, hook MustFunc) { m := hookManagerCtx(ctx) m.postRollbackHooks = append(m.postRollbackHooks, hook) } -func AddPostCompleteHook(ctx context.Context, hook TxnFunc) { +func AddPostCompleteHook(ctx context.Context, hook MustFunc) { m := hookManagerCtx(ctx) m.postCompleteHooks = append(m.postCompleteHooks, hook) } diff --git a/pkg/txn/transaction.go b/pkg/txn/transaction.go index 0989e438e..751588eff 100644 --- a/pkg/txn/transaction.go +++ b/pkg/txn/transaction.go @@ -17,13 +17,14 @@ type DatabaseProvider interface { WithDatabase(ctx context.Context) (context.Context, error) } -type DatabaseProviderManager interface { - DatabaseProvider - Manager -} - +// TxnFunc is a function that is used in transaction hooks. +// It should return an error if something went wrong. type TxnFunc func(ctx context.Context) error +// MustFunc is a function that is used in transaction hooks. +// It does not return an error. +type MustFunc func(ctx context.Context) + // WithTxn executes fn in a transaction. If fn returns an error then // the transaction is rolled back. Otherwise it is committed. // Transaction is exclusive. Only one thread may run a transaction @@ -51,35 +52,44 @@ func WithReadTxn(ctx context.Context, m Manager, fn TxnFunc) error { return withTxn(ctx, m, fn, exclusive, execComplete) } -func withTxn(outerCtx context.Context, m Manager, fn TxnFunc, exclusive bool, execCompleteOnLocked bool) error { - ctx, err := begin(outerCtx, m, exclusive) +func withTxn(ctx context.Context, m Manager, fn TxnFunc, exclusive bool, execCompleteOnLocked bool) error { + // post-hooks should be executed with the outside context + txnCtx, err := begin(ctx, m, exclusive) if err != nil { return err } + hookMgr := hookManagerCtx(txnCtx) + defer func() { if p := recover(); p != nil { // a panic occurred, rollback and repanic - rollback(ctx, outerCtx, m) + rollback(txnCtx, m) panic(p) } if err != nil { // something went wrong, rollback - rollback(ctx, outerCtx, m) + rollback(txnCtx, m) + + // execute post-hooks with outside context + hookMgr.executePostRollbackHooks(ctx) if execCompleteOnLocked || !m.IsLocked(err) { - executePostCompleteHooks(ctx, outerCtx) + hookMgr.executePostCompleteHooks(ctx) } } else { // all good, commit - err = commit(ctx, outerCtx, m) - executePostCompleteHooks(ctx, outerCtx) + err = commit(txnCtx, m) + + // execute post-hooks with outside context + hookMgr.executePostCommitHooks(ctx) + hookMgr.executePostCompleteHooks(ctx) } }() - err = fn(ctx) + err = fn(txnCtx) return err } @@ -96,21 +106,23 @@ func begin(ctx context.Context, m Manager, exclusive bool) (context.Context, err return ctx, nil } -func commit(ctx context.Context, outerCtx context.Context, m Manager) error { +func commit(ctx context.Context, m Manager) error { + hookMgr := hookManagerCtx(ctx) + if err := hookMgr.executePreCommitHooks(ctx); err != nil { + return err + } + if err := m.Commit(ctx); err != nil { return err } - executePostCommitHooks(ctx, outerCtx) return nil } -func rollback(ctx context.Context, outerCtx context.Context, m Manager) { +func rollback(ctx context.Context, m Manager) { if err := m.Rollback(ctx); err != nil { return } - - executePostRollbackHooks(ctx, outerCtx) } // WithDatabase executes fn with the context provided by p.WithDatabase. diff --git a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx index bb6d5131f..c5b0f36c1 100644 --- a/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsSystemPanel.tsx @@ -15,8 +15,11 @@ import { VideoPreviewInput, VideoPreviewSettingsInput, } from "./GeneratePreviewOptions"; +import { useIntl } from "react-intl"; export const SettingsConfigurationPanel: React.FC = () => { + const intl = useIntl(); + const { general, loading, error, saveGeneral } = React.useContext(SettingStateContext); @@ -94,20 +97,23 @@ export const SettingsConfigurationPanel: React.FC = () => { return GQL.HashAlgorithm.Md5; } + function blobStorageTypeToID(value: GQL.BlobsStorageType | undefined) { + switch (value) { + case GQL.BlobsStorageType.Database: + return "blobs_storage_type.database"; + case GQL.BlobsStorageType.Filesystem: + return "blobs_storage_type.filesystem"; + } + + return "blobs_storage_type.database"; + } + if (error) return

{error.message}

; if (loading) return ; return ( <> - saveGeneral({ databasePath: v })} - /> - { /> + + saveGeneral({ databasePath: v })} + /> + + saveGeneral({ blobsStorage: v as GQL.BlobsStorageType }) + } + > + {Object.values(GQL.BlobsStorageType).map((q) => ( + + ))} + + saveGeneral({ blobsPath: v })} + /> + + = ({ dryRun: false, }); + const [migrateBlobsOptions, setMigrateBlobsOptions] = + useState({ + deleteOld: true, + }); + + const [migrateSceneScreenshotsOptions, setMigrateSceneScreenshotsOptions] = + useState({ + deleteFiles: false, + overwriteExisting: false, + }); + type DialogOpenState = typeof dialogOpen; function setDialogOpen(s: Partial) { @@ -256,6 +269,42 @@ export const DataManagementTasks: React.FC = ({ } } + async function onMigrateSceneScreenshots() { + try { + await mutateMigrateSceneScreenshots(migrateSceneScreenshotsOptions); + Toast.success({ + content: intl.formatMessage( + { id: "config.tasks.added_job_to_queue" }, + { + operation_name: intl.formatMessage({ + id: "actions.migrate_scene_screenshots", + }), + } + ), + }); + } catch (err) { + Toast.error(err); + } + } + + async function onMigrateBlobs() { + try { + await mutateMigrateBlobs(migrateBlobsOptions); + Toast.success({ + content: intl.formatMessage( + { id: "config.tasks.added_job_to_queue" }, + { + operation_name: intl.formatMessage({ + id: "actions.migrate_blobs", + }), + } + ), + }); + } catch (err) { + Toast.error(err); + } + } + async function onExport() { try { await mutateMetadataExport(); @@ -507,6 +556,69 @@ export const DataManagementTasks: React.FC = ({ + +
+ + + + + + setMigrateBlobsOptions({ ...migrateBlobsOptions, deleteOld: v }) + } + /> +
+ +
+ + + + + + setMigrateSceneScreenshotsOptions({ + ...migrateSceneScreenshotsOptions, + overwriteExisting: v, + }) + } + /> + + + setMigrateSceneScreenshotsOptions({ + ...migrateSceneScreenshotsOptions, + deleteFiles: v, + }) + } + /> +
); diff --git a/ui/v2.5/src/components/Settings/Tasks/GenerateOptions.tsx b/ui/v2.5/src/components/Settings/Tasks/GenerateOptions.tsx index 5e8864f26..4436bf179 100644 --- a/ui/v2.5/src/components/Settings/Tasks/GenerateOptions.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/GenerateOptions.tsx @@ -26,6 +26,12 @@ export const GenerateOptions: React.FC = ({ return ( <> + setOptions({ covers: v })} + /> { function getDefaultGenerateOptions(): GQL.GenerateMetadataInput { return { + covers: true, sprites: true, phashes: true, previews: true, diff --git a/ui/v2.5/src/components/Settings/Tasks/ScanOptions.tsx b/ui/v2.5/src/components/Settings/Tasks/ScanOptions.tsx index 2f9497ee3..f8d44dc26 100644 --- a/ui/v2.5/src/components/Settings/Tasks/ScanOptions.tsx +++ b/ui/v2.5/src/components/Settings/Tasks/ScanOptions.tsx @@ -12,6 +12,7 @@ export const ScanOptions: React.FC = ({ setOptions: setOptionsState, }) => { const { + scanGenerateCovers, scanGeneratePreviews, scanGenerateImagePreviews, scanGenerateSprites, @@ -25,6 +26,12 @@ export const ScanOptions: React.FC = ({ return ( <> + setOptions({ scanGenerateCovers: v })} + /> { const [databaseFile, setDatabaseFile] = useState(""); const [generatedLocation, setGeneratedLocation] = useState(""); const [cacheLocation, setCacheLocation] = useState(""); + const [blobsLocation, setBlobsLocation] = useState("blobs"); const [loading, setLoading] = useState(false); const [setupError, setSetupError] = useState(""); @@ -49,6 +50,7 @@ export const Setup: React.FC = () => { const [showGeneratedSelectDialog, setShowGeneratedSelectDialog] = useState(false); const [showCacheSelectDialog, setShowCacheSelectDialog] = useState(false); + const [showBlobsDialog, setShowBlobsDialog] = useState(false); const { data: systemStatus, loading: statusLoading } = useSystemStatus(); @@ -253,6 +255,22 @@ export const Setup: React.FC = () => { return ; } + function onBlobsClosed(d?: string) { + if (d) { + setBlobsLocation(d); + } + + setShowBlobsDialog(false); + } + + function maybeRenderBlobsSelectDialog() { + if (!showBlobsDialog) { + return; + } + + return ; + } + function maybeRenderGenerated() { if (!configuration?.general.generatedPath) { return ( @@ -351,6 +369,56 @@ export const Setup: React.FC = () => { } } + function maybeRenderBlobs() { + if (!configuration?.general.blobsPath) { + return ( + +

+ +

+

+ {chunks}, + }} + /> +

+

+ {chunks}, + strong: (chunks: string) => {chunks}, + }} + /> +

+ + ) => + setBlobsLocation(e.currentTarget.value) + } + /> + + + + + + ); + } + } + function renderSetPaths() { return ( <> @@ -410,6 +478,7 @@ export const Setup: React.FC = () => { {maybeRenderGenerated()} {maybeRenderCache()} + {maybeRenderBlobs()}
@@ -474,6 +543,7 @@ export const Setup: React.FC = () => { databaseFile, generatedLocation, cacheLocation, + blobsLocation, stashes, }); // Set lastNoteSeen to hide release notes dialog @@ -556,6 +626,18 @@ export const Setup: React.FC = () => { })} +
+ +
+
+ + {blobsLocation !== "" + ? blobsLocation + : intl.formatMessage({ + id: "setup.confirm.default_blobs_location", + })} + +
@@ -739,6 +821,7 @@ export const Setup: React.FC = () => { {maybeRenderGeneratedSelectDialog()} {maybeRenderCacheSelectDialog()} + {maybeRenderBlobsSelectDialog()}

diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 421f9fd98..87295e0c1 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -1215,6 +1215,20 @@ export const mutateMigrateHashNaming = () => mutation: GQL.MigrateHashNamingDocument, }); +export const mutateMigrateSceneScreenshots = ( + input: GQL.MigrateSceneScreenshotsInput +) => + client.mutate({ + mutation: GQL.MigrateSceneScreenshotsDocument, + variables: { input }, + }); + +export const mutateMigrateBlobs = (input: GQL.MigrateBlobsInput) => + client.mutate({ + mutation: GQL.MigrateBlobsDocument, + variables: { input }, + }); + export const mutateMetadataExport = () => client.mutate({ mutation: GQL.MetadataExportDocument, diff --git a/ui/v2.5/src/docs/en/Changelog/v0200.md b/ui/v2.5/src/docs/en/Changelog/v0200.md index 9a3692d75..b1035facb 100644 --- a/ui/v2.5/src/docs/en/Changelog/v0200.md +++ b/ui/v2.5/src/docs/en/Changelog/v0200.md @@ -1,6 +1,17 @@ ##### 💥 Note: The cache directory is now required if using HLS/DASH streaming. Please set the cache directory in the System Settings page. +##### 💥 Note: The image data subsystem has been reworked in this release. Existing systems will have their storage system set to `Database`, which stores all image data in the database. This can be changed in the System Settings page. + +A migration is required to change the storage system, and can be accessed from the Tasks page. + +The `Database` storage system is not recommended for large libraries, as it can cause performance issues. The `Filesystem` storage system is recommended for large libraries, and is the default for new systems. + +##### 💥 Note: the `generated/screenshots` jpg files are now considered legacy. These files can be migrated into the blob storage system by running the `Migrate Screenshots` task from the Tasks page. + +Once migrated, these files can be deleted. The files can be optionally deleted during the migration. + ### ✨ New Features +* Added `Is Missing Cover` scene filter criterion. ([#3187](https://github.com/stashapp/stash/pull/3187)) * Added Chapters to Galleries. ([#3289](https://github.com/stashapp/stash/pull/3289)) * Added button to tagger scene cards to view scene sprite. ([#3536](https://github.com/stashapp/stash/pull/3536)) * Added hardware acceleration support (for a limited number of encoders) for transcoding. ([#3419](https://github.com/stashapp/stash/pull/3419)) @@ -14,6 +25,8 @@ * Added toggleable favorite button to Performer cards. ([#3369](https://github.com/stashapp/stash/pull/3369)) ### 🎨 Improvements +* Scene cover generation is now optional during scanning, and can be generated using the Generate task. ([#3187](https://github.com/stashapp/stash/pull/3187)) +* Overhauled the image blob storage system and added filesystem-based blob storage. ([#3187](https://github.com/stashapp/stash/pull/3187)) * Overhauled filtering interface to allow setting filter criteria from a single dialog. ([#3515](https://github.com/stashapp/stash/pull/3515)) * Removed upper limit on page size. ([#3544](https://github.com/stashapp/stash/pull/3544)) * Anonymise task now obfuscates Marker titles. ([#3542](https://github.com/stashapp/stash/pull/3542)) diff --git a/ui/v2.5/src/docs/en/ReleaseNotes/index.ts b/ui/v2.5/src/docs/en/ReleaseNotes/index.ts index 3d10ee138..9eb1a8cb9 100644 --- a/ui/v2.5/src/docs/en/ReleaseNotes/index.ts +++ b/ui/v2.5/src/docs/en/ReleaseNotes/index.ts @@ -10,7 +10,7 @@ export interface IReleaseNotes { export const releaseNotes: IReleaseNotes[] = [ { - date: 20230224, + date: 20230301, version: "v0.20.0", content: v0200, }, diff --git a/ui/v2.5/src/docs/en/ReleaseNotes/v0200.md b/ui/v2.5/src/docs/en/ReleaseNotes/v0200.md index 4e5e50476..c14895101 100644 --- a/ui/v2.5/src/docs/en/ReleaseNotes/v0200.md +++ b/ui/v2.5/src/docs/en/ReleaseNotes/v0200.md @@ -1 +1,7 @@ -The cache directory is now required if using HLS/DASH streaming. Please set the cache directory in the System Settings page. \ No newline at end of file +The image data subsystem has been reworked in this release. Existing systems will have their storage system set to `Database`, which stores all image data in the database. This can be changed in the System Settings page. **Note:** a migration is required to change the storage system, and can be accessed from the Tasks page. + +The `Database` storage system is not recommended for large libraries, as it can cause performance issues. The `Filesystem` storage system is recommended for large libraries, and is the default for new systems. + +**Important:** the `generated/screenshots` jpg files are considered legacy. These files can be migrated into the blob storage system by running the `Migrate Screenshots` task from the Tasks page. Once migrated, these files can be deleted. The files can be optionally deleted during the migration. + +The cache directory is now required if using HLS/DASH streaming. Please set the cache directory in the System Settings page. diff --git a/ui/v2.5/src/locales/en-GB.json b/ui/v2.5/src/locales/en-GB.json index 34120dd06..acdff64e8 100644 --- a/ui/v2.5/src/locales/en-GB.json +++ b/ui/v2.5/src/locales/en-GB.json @@ -61,6 +61,8 @@ "merge": "Merge", "merge_from": "Merge from", "merge_into": "Merge into", + "migrate_blobs": "Migrate Blobs", + "migrate_scene_screenshots": "Migrate Scene Screenshots", "next_action": "Next", "not_running": "not running", "open_in_external_player": "Open in external player", @@ -131,6 +133,10 @@ "birthdate": "Birthdate", "bitrate": "Bit Rate", "between_and": "and", + "blobs_storage_type": { + "database": "Database", + "filesystem": "Filesystem" + }, "captions": "Captions", "career_length": "Career Length", "chapters": "Chapters", @@ -258,6 +264,14 @@ "description": "Directory location for SQLite database file backups", "heading": "Backup Directory Path" }, + "blobs_storage": { + "heading": "Binary data storage type", + "description": "Where to store binary data such as scene covers, performer, studio and tag images. After changing this value, the existing data must be migrated using the Migrate Blobs tasks. See Tasks page for migration." + }, + "blobs_path": { + "heading": "Binary data filesystem path", + "description": "Where in the filesystem to store binary data. Applicable only when using the Filesystem blob storage type. WARNING: changing this requires manually moving existing data." + }, "cache_location": "Directory location of the cache. Required if streaming using HLS (such as on Apple devices) or DASH.", "cache_path_head": "Cache Path", "calculate_md5_and_ohash_desc": "Calculate MD5 checksum in addition to oshash. Enabling will cause initial scans to be slower. File naming hash must be set to oshash to disable MD5 calculation.", @@ -270,6 +284,7 @@ "create_galleries_from_folders_label": "Create galleries from folders containing images", "gallery_cover_regex_desc": "Regexp used to identify an image as gallery cover", "gallery_cover_regex_label": "Gallery cover pattern", + "database": "Database", "db_path_head": "Database Path", "directory_locations_to_your_content": "Directory locations to your content", "excluded_image_gallery_patterns_desc": "Regexps of image and gallery files/paths to exclude from Scan and add to Clean", @@ -411,6 +426,7 @@ "generate_previews_during_scan_tooltip": "Generate animated WebP previews, only required if Preview Type is set to Animated Image.", "generate_sprites_during_scan": "Generate scrubber sprites", "generate_thumbnails_during_scan": "Generate thumbnails for images", + "generate_video_covers_during_scan": "Generate scene covers", "generate_video_previews_during_scan": "Generate previews", "generate_video_previews_during_scan_tooltip": "Generate video previews which play when hovering over a scene", "generated_content": "Generated Content", @@ -438,7 +454,16 @@ "incremental_import": "Incremental import from a supplied export zip file.", "job_queue": "Task Queue", "maintenance": "Maintenance", + "migrate_blobs": { + "delete_old": "Delete old data", + "description": "Migrate blobs to the current blob storage system. This migration should be run after changing the blob storage system. Can optionally delete the old data after migration." + }, "migrate_hash_files": "Used after changing the Generated file naming hash to rename existing generated files to the new hash format.", + "migrate_scene_screenshots": { + "delete_files": "Delete screenshot files", + "description": "Migrate scene screenshots into the new blob storage system. This migration should be run after migrating an existing system to 0.20. Can optionally delete the old screenshots after migration.", + "overwrite_existing": "Overwrite existing blobs with screenshot data" + }, "migrations": "Migrations", "only_dry_run": "Only perform a dry run. Don't remove anything", "plugin_tasks": "Plugin Tasks", @@ -770,6 +795,7 @@ "destination": "Reassign to" }, "scene_gen": { + "covers": "Scene covers", "force_transcodes": "Force Transcode generation", "force_transcodes_tooltip": "By default, transcodes are only generated when the video file is not supported in the browser. When enabled, transcodes will be generated even when the video file appears to be supported in the browser.", "image_previews": "Animated Image Previews", @@ -1049,9 +1075,11 @@ "setup": { "confirm": { "almost_ready": "We're almost ready to complete the configuration. Please confirm the following settings. You can click back to change anything incorrect. If everything looks good, click Confirm to create your system.", + "blobs_directory": "Binary data directory", "cache_directory": "Cache directory", "configuration_file_location": "Configuration file location:", "database_file_path": "Database file path", + "default_blobs_location": "", "default_cache_location": "/cache", "default_db_location": "/stash-go.sqlite", "default_generated_content_location": "/generated", @@ -1091,6 +1119,7 @@ "description": "Next up, we need to determine where to find your porn collection, and where to store the Stash database, generated files and cache files. These settings can be changed later if needed.", "path_to_cache_directory_empty_for_default": "path to cache directory (empty for default)", "path_to_generated_directory_empty_for_default": "path to generated directory (empty for default)", + "path_to_blobs_directory_empty_for_database": "path to blobs directory (empty to use database)", "set_up_your_paths": "Set up your paths", "stash_alert": "No library paths have been selected. No media will be able to be scanned into Stash. Are you sure?", "where_can_stash_store_cache_files": "Where can Stash store cache files?", @@ -1098,6 +1127,9 @@ "where_can_stash_store_its_database": "Where can Stash store its database?", "where_can_stash_store_its_database_description": "Stash uses an SQLite database to store your porn metadata. By default, this will be created as stash-go.sqlite in the directory containing your config file. If you want to change this, please enter an absolute or relative (to the current working directory) filename.", "where_can_stash_store_its_database_warning": "WARNING: storing the database on a different system to where Stash is run from (e.g. storing the database on a NAS while running the Stash server on another computer) is unsupported! SQLite is not intended for use across a network, and attempting to do so can very easily cause your entire database to become corrupted.", + "where_can_stash_store_blobs": "Where can Stash store database binary data?", + "where_can_stash_store_blobs_description": "Stash can store binary data such as scene covers, performer, studio and tag images either in the database or in the filesystem. By default, it will store this data in the filesystem in the subdirectory blobs. If you want to change this, please enter an absolute or relative (to the current working directory) path. Stash will create this directory if it does not already exist.", + "where_can_stash_store_blobs_description_addendum": "Alternatively, if you want to store this data in the database, you can leave this field blank. Note: This will increase the size of your database file, and will increase database migration times.", "where_can_stash_store_its_generated_content": "Where can Stash store its generated content?", "where_can_stash_store_its_generated_content_description": "In order to provide thumbnails, previews and sprites, Stash generates images and videos. This also includes transcodes for unsupported file formats. By default, Stash will create a generated directory within the directory containing your config file. If you want to change where this generated media will be stored, please enter an absolute or relative (to the current working directory) path. Stash will create this directory if it does not already exist.", "where_is_your_porn_located": "Where is your porn located?", diff --git a/ui/v2.5/src/models/list-filter/criteria/is-missing.ts b/ui/v2.5/src/models/list-filter/criteria/is-missing.ts index 75588fa02..5ea971d2c 100644 --- a/ui/v2.5/src/models/list-filter/criteria/is-missing.ts +++ b/ui/v2.5/src/models/list-filter/criteria/is-missing.ts @@ -33,6 +33,7 @@ export const SceneIsMissingCriterionOption = new IsMissingCriterionOptionClass( "is_missing", [ "title", + "cover", "details", "url", "date",