From f7676350808189386e80e6622223986ba30914a3 Mon Sep 17 00:00:00 2001 From: JoeSmithStarkers <60503487+JoeSmithStarkers@users.noreply.github.com> Date: Thu, 23 Feb 2023 14:38:02 +1100 Subject: [PATCH] Added the ability to do Sequential Scans (#3378) * Added the ability to do Seqential Scans * Modify pkg/txn to run hooks with the outer context, instead of the context that was in a transaction * update in application manual --- internal/manager/config/config.go | 8 ++++++ internal/manager/task_scan.go | 31 +++++++++++++++++---- pkg/txn/hooks.go | 12 ++++---- pkg/txn/transaction.go | 23 ++++++++------- ui/v2.5/src/docs/en/Changelog/v0200.md | 1 + ui/v2.5/src/docs/en/Manual/Configuration.md | 1 + 6 files changed, 52 insertions(+), 24 deletions(-) diff --git a/internal/manager/config/config.go b/internal/manager/config/config.go index 7c954864d..bea1381e4 100644 --- a/internal/manager/config/config.go +++ b/internal/manager/config/config.go @@ -69,6 +69,9 @@ const ( ParallelTasks = "parallel_tasks" parallelTasksDefault = 1 + SequentialScanning = "sequential_scanning" + SequentialScanningDefault = false + PreviewPreset = "preview_preset" PreviewAudio = "preview_audio" @@ -649,6 +652,10 @@ func (i *Instance) GetVideoFileNamingAlgorithm() models.HashAlgorithm { return models.HashAlgorithm(ret) } +func (i *Instance) GetSequentialScanning() bool { + return i.getBool(SequentialScanning) +} + func (i *Instance) GetGalleryCoverRegex() string { var regexString = i.getString(GalleryCoverRegex) @@ -1464,6 +1471,7 @@ func (i *Instance) setDefaultValues(write bool) error { i.main.SetDefault(Port, portDefault) i.main.SetDefault(ParallelTasks, parallelTasksDefault) + i.main.SetDefault(SequentialScanning, SequentialScanningDefault) i.main.SetDefault(PreviewSegmentDuration, previewSegmentDurationDefault) i.main.SetDefault(PreviewSegments, previewSegmentsDefault) i.main.SetDefault(PreviewExcludeStart, previewExcludeStartDefault) diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index d4209aa44..1d8419dcc 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -410,10 +410,11 @@ func (g *sceneGenerators) Generate(ctx context.Context, s *models.Scene, f *file path := f.Path config := instance.Config fileNamingAlgorithm := config.GetVideoFileNamingAlgorithm() + sequentialScanning := config.GetSequentialScanning() if t.ScanGenerateSprites { progress.AddTotal(1) - g.taskQueue.Add(fmt.Sprintf("Generating sprites for %s", path), func(ctx context.Context) { + spriteFn := func(ctx context.Context) { taskSprite := GenerateSpriteTask{ Scene: *s, Overwrite: overwrite, @@ -421,12 +422,18 @@ func (g *sceneGenerators) Generate(ctx context.Context, s *models.Scene, f *file } taskSprite.Start(ctx) progress.Increment() - }) + } + + if sequentialScanning { + spriteFn(ctx) + } else { + g.taskQueue.Add(fmt.Sprintf("Generating sprites for %s", path), spriteFn) + } } if t.ScanGeneratePhashes { progress.AddTotal(1) - g.taskQueue.Add(fmt.Sprintf("Generating phash for %s", path), func(ctx context.Context) { + phashFn := func(ctx context.Context) { taskPhash := GeneratePhashTask{ File: f, fileNamingAlgorithm: fileNamingAlgorithm, @@ -436,12 +443,18 @@ func (g *sceneGenerators) Generate(ctx context.Context, s *models.Scene, f *file } taskPhash.Start(ctx) progress.Increment() - }) + } + + if sequentialScanning { + phashFn(ctx) + } else { + g.taskQueue.Add(fmt.Sprintf("Generating phash for %s", path), phashFn) + } } if t.ScanGeneratePreviews { progress.AddTotal(1) - g.taskQueue.Add(fmt.Sprintf("Generating preview for %s", path), func(ctx context.Context) { + previewsFn := func(ctx context.Context) { options := getGeneratePreviewOptions(GeneratePreviewOptionsInput{}) g := &generate.Generator{ @@ -463,7 +476,13 @@ func (g *sceneGenerators) Generate(ctx context.Context, s *models.Scene, f *file } taskPreview.Start(ctx) progress.Increment() - }) + } + + if sequentialScanning { + previewsFn(ctx) + } else { + g.taskQueue.Add(fmt.Sprintf("Generating preview for %s", path), previewsFn) + } } return nil diff --git a/pkg/txn/hooks.go b/pkg/txn/hooks.go index 13cc85d05..5f36d7def 100644 --- a/pkg/txn/hooks.go +++ b/pkg/txn/hooks.go @@ -35,19 +35,19 @@ func executeHooks(ctx context.Context, hooks []TxnFunc) { } } -func executePostCommitHooks(ctx context.Context) { +func executePostCommitHooks(ctx context.Context, outerCtx context.Context) { m := hookManagerCtx(ctx) - executeHooks(ctx, m.postCommitHooks) + executeHooks(outerCtx, m.postCommitHooks) } -func executePostRollbackHooks(ctx context.Context) { +func executePostRollbackHooks(ctx context.Context, outerCtx context.Context) { m := hookManagerCtx(ctx) - executeHooks(ctx, m.postRollbackHooks) + executeHooks(outerCtx, m.postRollbackHooks) } -func executePostCompleteHooks(ctx context.Context) { +func executePostCompleteHooks(ctx context.Context, outerCtx context.Context) { m := hookManagerCtx(ctx) - executeHooks(ctx, m.postCompleteHooks) + executeHooks(outerCtx, m.postCompleteHooks) } func AddPostCommitHook(ctx context.Context, hook TxnFunc) { diff --git a/pkg/txn/transaction.go b/pkg/txn/transaction.go index 2a78da721..0989e438e 100644 --- a/pkg/txn/transaction.go +++ b/pkg/txn/transaction.go @@ -51,9 +51,8 @@ func WithReadTxn(ctx context.Context, m Manager, fn TxnFunc) error { return withTxn(ctx, m, fn, exclusive, execComplete) } -func withTxn(ctx context.Context, m Manager, fn TxnFunc, exclusive bool, execCompleteOnLocked bool) error { - var err error - ctx, err = begin(ctx, m, exclusive) +func withTxn(outerCtx context.Context, m Manager, fn TxnFunc, exclusive bool, execCompleteOnLocked bool) error { + ctx, err := begin(outerCtx, m, exclusive) if err != nil { return err } @@ -61,21 +60,21 @@ func withTxn(ctx context.Context, m Manager, fn TxnFunc, exclusive bool, execCom defer func() { if p := recover(); p != nil { // a panic occurred, rollback and repanic - rollback(ctx, m) + rollback(ctx, outerCtx, m) panic(p) } if err != nil { // something went wrong, rollback - rollback(ctx, m) + rollback(ctx, outerCtx, m) if execCompleteOnLocked || !m.IsLocked(err) { - executePostCompleteHooks(ctx) + executePostCompleteHooks(ctx, outerCtx) } } else { // all good, commit - err = commit(ctx, m) - executePostCompleteHooks(ctx) + err = commit(ctx, outerCtx, m) + executePostCompleteHooks(ctx, outerCtx) } }() @@ -97,21 +96,21 @@ func begin(ctx context.Context, m Manager, exclusive bool) (context.Context, err return ctx, nil } -func commit(ctx context.Context, m Manager) error { +func commit(ctx context.Context, outerCtx context.Context, m Manager) error { if err := m.Commit(ctx); err != nil { return err } - executePostCommitHooks(ctx) + executePostCommitHooks(ctx, outerCtx) return nil } -func rollback(ctx context.Context, m Manager) { +func rollback(ctx context.Context, outerCtx context.Context, m Manager) { if err := m.Rollback(ctx); err != nil { return } - executePostRollbackHooks(ctx) + executePostRollbackHooks(ctx, outerCtx) } // WithDatabase executes fn with the context provided by p.WithDatabase. diff --git a/ui/v2.5/src/docs/en/Changelog/v0200.md b/ui/v2.5/src/docs/en/Changelog/v0200.md index 1fb87f0f9..3314abd69 100644 --- a/ui/v2.5/src/docs/en/Changelog/v0200.md +++ b/ui/v2.5/src/docs/en/Changelog/v0200.md @@ -1,4 +1,5 @@ ### ✨ New Features +* Add configuration option to perform generation operations sequentially after scanning a new video file. ([#3378](https://github.com/stashapp/stash/pull/3378)) * Optionally show range in generated funscript heatmaps. ([#3373](https://github.com/stashapp/stash/pull/3373)) * Show funscript heatmaps in scene player scrubber. ([#3374](https://github.com/stashapp/stash/pull/3374)) * Support customising the filename regex used for determining the gallery cover image. ([#3391](https://github.com/stashapp/stash/pull/3391)) diff --git a/ui/v2.5/src/docs/en/Manual/Configuration.md b/ui/v2.5/src/docs/en/Manual/Configuration.md index 8928a0789..1cb85ddce 100644 --- a/ui/v2.5/src/docs/en/Manual/Configuration.md +++ b/ui/v2.5/src/docs/en/Manual/Configuration.md @@ -132,6 +132,7 @@ These options are typically not exposed in the UI and must be changed manually i | `gallery_cover_regex` | The regex responsible for selecting images as gallery covers | | `proxy` | The url of a HTTP(S) proxy to be used when stash makes calls to online services Example: https://user:password@my.proxy:8080 | | `no_proxy` | A list of domains for which the proxy must not be used. Default is all local LAN: localhost,127.0.0.1,192.168.0.0/16,10.0.0.0/8,172.16.0.0/12 | +| `sequential_scanning` | Modifies behaviour of the scanning functionality to generate support files (previews/sprites/phash) at the same time as fingerprinting/screenshotting. Useful when scanning cached remote files. | ### Custom served folders