Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Hoist context, enable errchkjson (#2488)

Browse Source 
* Make the script scraper context-aware

Connect the context to the command execution. This means command
execution can be aborted if the context is canceled. The context is
usually bound to user-interaction, i.e., a scraper operation issued
by the user. Hence, it seems correct to abort a command if the user
aborts.

* Enable errchkjson

Some json marshal calls are *safe* in that they can never fail. This is
conditional on the types of the the data being encoded. errchkjson finds
those calls which are unsafe, and also not checked for errors.

Add logging warnings to the place where unsafe encodings might happen.
This can help uncover usage bugs early in stash if they are tripped,
making debugging easier.

While here, keep the checker enabled in the linter to capture future
uses of json marshalling.

* Pass the context for zip file scanning.

* Pass the context in scanning

* Pass context, replace context.TODO()

Where applicable, pass the context down toward the lower functions in
the call stack. Replace uses of context.TODO() with the passed context.

This makes the code more context-aware, and you can rely on aborting
contexts to clean up subsystems to a far greater extent now.

I've left the cases where there is a context in a struct. My gut feeling
is that they have solutions that are nice, but they require more deep
thinking to unveil how to handle it.

* Remove context from task-structs

As a rule, contexts are better passed explicitly to functions than they
are passed implicitly via structs. In the case of tasks, we already
have a valid context in scope when creating the struct, so remove ctx
from the struct and use the scoped context instead.

With this change it is clear that the scanning functions are under a
context, and the task-starting caller has jurisdiction over the context
and its lifetime. A reader of the code don't have to figure out where
the context are coming from anymore.

While here, connect context.TODO() to the newly scoped context in most
of the scan code.

* Remove context from autotag struct too

* Make more context-passing explicit

In all of these cases, there is an applicable context which is close
in the call-tree. Hook up to this context.

* Simplify context passing in manager

The managers context handling generally wants to use an outer context
if applicable. However, the code doesn't pass it explicitly, but stores
it in a struct. Pull out the context from the struct and use it to
explicitly pass it.

At a later point in time, we probably want to handle this by handing
over the job to a different (program-lifetime) context for background
jobs, but this will do for a start.
This commit is contained in:
SmallCoccinelle
2022-04-15 03:34:53 +02:00
committed by GitHub
parent a7beeb32b0
commit 401660e6a3
24 changed files with 141 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.golangci.yml
View File

@@ -20,7 +20,7 @@ linters:
# Linters added by the stash project. # Linters added by the stash project.
# - contextcheck # - contextcheck
- dogsled - dogsled
# - errchkjson - errchkjson
- errorlint - errorlint
# - exhaustive # - exhaustive
- exportloopref - exportloopref

2
internal/api/resolver_mutation_metadata.go
View File

@@ -61,7 +61,7 @@ func (r *mutationResolver) ExportObjects(ctx context.Context, input models.Expor
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
t.Start(&wg) t.Start(ctx, &wg)
if t.DownloadHash != "" { if t.DownloadHash != "" {
baseURL, _ := ctx.Value(BaseURLCtxKey).(string) baseURL, _ := ctx.Value(BaseURLCtxKey).(string)

8
internal/manager/manager.go
View File

@@ -121,7 +121,7 @@ func Initialize() *singleton {
logger.Warnf("config file %snot found. Assuming new system...", cfgFile) logger.Warnf("config file %snot found. Assuming new system...", cfgFile)
} }
if err = initFFMPEG(); err != nil { if err = initFFMPEG(ctx); err != nil {
logger.Warnf("could not initialize FFMPEG subsystem: %v", err) logger.Warnf("could not initialize FFMPEG subsystem: %v", err)
} }
@@ -189,9 +189,7 @@ func initProfiling(cpuProfilePath string) {
} }
} }
func initFFMPEG() error { func initFFMPEG(ctx context.Context) error {
ctx := context.TODO()
// only do this if we have a config file set // only do this if we have a config file set
if instance.Config.GetConfigFile() != "" { if instance.Config.GetConfigFile() != "" {
// use same directory as config path // use same directory as config path
@@ -404,7 +402,7 @@ func (s *singleton) Setup(ctx context.Context, input models.SetupInput) error {
s.Config.FinalizeSetup() s.Config.FinalizeSetup()
if err := initFFMPEG(); err != nil { if err := initFFMPEG(ctx); err != nil {
return fmt.Errorf("error initializing FFMPEG subsystem: %v", err) return fmt.Errorf("error initializing FFMPEG subsystem: %v", err)
} }

12
internal/manager/manager_tasks.go
View File

@@ -108,7 +108,7 @@ func (s *singleton) Export(ctx context.Context) (int, error) {
full: true, full: true,
fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), fileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(),
} }
task.Start(&wg) task.Start(ctx, &wg)
}) })
return s.JobManager.Add(ctx, "Exporting...", j), nil return s.JobManager.Add(ctx, "Exporting...", j), nil
@@ -192,7 +192,7 @@ func (s *singleton) generateScreenshot(ctx context.Context, sceneId string, at *
} }
var scene *models.Scene var scene *models.Scene
if err := s.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := s.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
scene, err = r.Scene().Find(sceneIdInt) scene, err = r.Scene().Find(sceneIdInt)
return err return err
@@ -241,7 +241,7 @@ func (s *singleton) MigrateHash(ctx context.Context) int {
logger.Infof("Migrating generated files for %s naming hash", fileNamingAlgo.String()) logger.Infof("Migrating generated files for %s naming hash", fileNamingAlgo.String())
var scenes []*models.Scene var scenes []*models.Scene
if err := s.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := s.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
scenes, err = r.Scene().All() scenes, err = r.Scene().All()
return err return err
@@ -303,7 +303,7 @@ func (s *singleton) StashBoxBatchPerformerTag(ctx context.Context, input models.
// This is why we mark this section nolint. In principle, we should look to // This is why we mark this section nolint. In principle, we should look to
// rewrite the section at some point, to avoid the linter warning. // rewrite the section at some point, to avoid the linter warning.
if len(input.PerformerIds) > 0 { //nolint:gocritic if len(input.PerformerIds) > 0 { //nolint:gocritic
if err := s.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := s.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
performerQuery := r.Performer() performerQuery := r.Performer()
for _, performerID := range input.PerformerIds { for _, performerID := range input.PerformerIds {
@@ -343,7 +343,7 @@ func (s *singleton) StashBoxBatchPerformerTag(ctx context.Context, input models.
// However, this doesn't really help with readability of the current section. Mark it // However, this doesn't really help with readability of the current section. Mark it
// as nolint for now. In the future we'd like to rewrite this code by factoring some of // as nolint for now. In the future we'd like to rewrite this code by factoring some of
// this into separate functions. // this into separate functions.
if err := s.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := s.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
performerQuery := r.Performer() performerQuery := r.Performer()
var performers []*models.Performer var performers []*models.Performer
var err error var err error
@@ -384,7 +384,7 @@ func (s *singleton) StashBoxBatchPerformerTag(ctx context.Context, input models.
for _, task := range tasks { for _, task := range tasks {
wg.Add(1) wg.Add(1)
progress.ExecuteTask(task.Description(), func() { progress.ExecuteTask(task.Description(), func() {
task.Start() task.Start(ctx)
wg.Done() wg.Done()
}) })

66
internal/manager/task_autotag.go
View File

@@ -55,13 +55,12 @@ func (j *autoTagJob) autoTagFiles(ctx context.Context, progress *job.Progress, p
performers: performers, performers: performers,
studios: studios, studios: studios,
tags: tags, tags: tags,
ctx: ctx,
progress: progress, progress: progress,
txnManager: j.txnManager, txnManager: j.txnManager,
cache: &j.cache, cache: &j.cache,
} }
t.process() t.process(ctx)
} }
func (j *autoTagJob) autoTagSpecific(ctx context.Context, progress *job.Progress) { func (j *autoTagJob) autoTagSpecific(ctx context.Context, progress *job.Progress) {
@@ -74,7 +73,7 @@ func (j *autoTagJob) autoTagSpecific(ctx context.Context, progress *job.Progress
studioCount := len(studioIds) studioCount := len(studioIds)
tagCount := len(tagIds) tagCount := len(tagIds)
if err := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
performerQuery := r.Performer() performerQuery := r.Performer()
studioQuery := r.Studio() studioQuery := r.Studio()
tagQuery := r.Tag() tagQuery := r.Tag()
@@ -124,7 +123,7 @@ func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progre
for _, performerId := range performerIds { for _, performerId := range performerIds {
var performers []*models.Performer var performers []*models.Performer
if err := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
performerQuery := r.Performer() performerQuery := r.Performer()
ignoreAutoTag := false ignoreAutoTag := false
perPage := -1 perPage := -1
@@ -162,7 +161,7 @@ func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progre
return nil return nil
} }
if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(r models.Repository) error {
if err := autotag.PerformerScenes(performer, paths, r.Scene(), &j.cache); err != nil { if err := autotag.PerformerScenes(performer, paths, r.Scene(), &j.cache); err != nil {
return err return err
} }
@@ -197,7 +196,7 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress,
for _, studioId := range studioIds { for _, studioId := range studioIds {
var studios []*models.Studio var studios []*models.Studio
if err := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
studioQuery := r.Studio() studioQuery := r.Studio()
ignoreAutoTag := false ignoreAutoTag := false
perPage := -1 perPage := -1
@@ -235,7 +234,7 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress,
return nil return nil
} }
if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(r models.Repository) error {
aliases, err := r.Studio().GetAliases(studio.ID) aliases, err := r.Studio().GetAliases(studio.ID)
if err != nil { if err != nil {
return err return err
@@ -274,7 +273,7 @@ func (j *autoTagJob) autoTagTags(ctx context.Context, progress *job.Progress, pa
for _, tagId := range tagIds { for _, tagId := range tagIds {
var tags []*models.Tag var tags []*models.Tag
if err := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
tagQuery := r.Tag() tagQuery := r.Tag()
ignoreAutoTag := false ignoreAutoTag := false
perPage := -1 perPage := -1
@@ -307,7 +306,7 @@ func (j *autoTagJob) autoTagTags(ctx context.Context, progress *job.Progress, pa
return nil return nil
} }
if err := j.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(r models.Repository) error {
aliases, err := r.Tag().GetAliases(tag.ID) aliases, err := r.Tag().GetAliases(tag.ID)
if err != nil { if err != nil {
return err return err
@@ -345,7 +344,6 @@ type autoTagFilesTask struct {
studios bool studios bool
tags bool tags bool
ctx context.Context
progress *job.Progress progress *job.Progress
txnManager models.TransactionManager txnManager models.TransactionManager
cache *match.Cache cache *match.Cache
@@ -467,8 +465,8 @@ func (t *autoTagFilesTask) getCount(r models.ReaderRepository) (int, error) {
return sceneCount + imageCount + galleryCount, nil return sceneCount + imageCount + galleryCount, nil
} }
func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error { func (t *autoTagFilesTask) processScenes(ctx context.Context, r models.ReaderRepository) error {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -485,7 +483,7 @@ func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error {
} }
for _, ss := range scenes { for _, ss := range scenes {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -500,7 +498,7 @@ func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
go tt.Start(&wg) go tt.Start(ctx, &wg)
wg.Wait() wg.Wait()
t.progress.Increment() t.progress.Increment()
@@ -520,8 +518,8 @@ func (t *autoTagFilesTask) processScenes(r models.ReaderRepository) error {
return nil return nil
} }
func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error { func (t *autoTagFilesTask) processImages(ctx context.Context, r models.ReaderRepository) error {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -538,7 +536,7 @@ func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error {
} }
for _, ss := range images { for _, ss := range images {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -553,7 +551,7 @@ func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
go tt.Start(&wg) go tt.Start(ctx, &wg)
wg.Wait() wg.Wait()
t.progress.Increment() t.progress.Increment()
@@ -573,8 +571,8 @@ func (t *autoTagFilesTask) processImages(r models.ReaderRepository) error {
return nil return nil
} }
func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error { func (t *autoTagFilesTask) processGalleries(ctx context.Context, r models.ReaderRepository) error {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -591,7 +589,7 @@ func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error {
} }
for _, ss := range galleries { for _, ss := range galleries {
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
return nil return nil
} }
@@ -606,7 +604,7 @@ func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error {
var wg sync.WaitGroup var wg sync.WaitGroup
wg.Add(1) wg.Add(1)
go tt.Start(&wg) go tt.Start(ctx, &wg)
wg.Wait() wg.Wait()
t.progress.Increment() t.progress.Increment()
@@ -626,8 +624,8 @@ func (t *autoTagFilesTask) processGalleries(r models.ReaderRepository) error {
return nil return nil
} }
func (t *autoTagFilesTask) process() { func (t *autoTagFilesTask) process(ctx context.Context) {
if err := t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
total, err := t.getCount(r) total, err := t.getCount(r)
if err != nil { if err != nil {
return err return err
@@ -638,21 +636,21 @@ func (t *autoTagFilesTask) process() {
logger.Infof("Starting autotag of %d files", total) logger.Infof("Starting autotag of %d files", total)
logger.Info("Autotagging scenes...") logger.Info("Autotagging scenes...")
if err := t.processScenes(r); err != nil { if err := t.processScenes(ctx, r); err != nil {
return err return err
} }
logger.Info("Autotagging images...") logger.Info("Autotagging images...")
if err := t.processImages(r); err != nil { if err := t.processImages(ctx, r); err != nil {
return err return err
} }
logger.Info("Autotagging galleries...") logger.Info("Autotagging galleries...")
if err := t.processGalleries(r); err != nil { if err := t.processGalleries(ctx, r); err != nil {
return err return err
} }
if job.IsCancelled(t.ctx) { if job.IsCancelled(ctx) {
logger.Info("Stopping due to user request") logger.Info("Stopping due to user request")
} }
@@ -673,9 +671,9 @@ type autoTagSceneTask struct {
cache *match.Cache cache *match.Cache
} }
func (t *autoTagSceneTask) Start(wg *sync.WaitGroup) { func (t *autoTagSceneTask) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.ScenePerformers(t.scene, r.Scene(), r.Performer(), t.cache); err != nil { if err := autotag.ScenePerformers(t.scene, r.Scene(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging scene performers for %s: %v", t.scene.Path, err) return fmt.Errorf("error tagging scene performers for %s: %v", t.scene.Path, err)
@@ -709,9 +707,9 @@ type autoTagImageTask struct {
cache *match.Cache cache *match.Cache
} }
func (t *autoTagImageTask) Start(wg *sync.WaitGroup) { func (t *autoTagImageTask) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.ImagePerformers(t.image, r.Image(), r.Performer(), t.cache); err != nil { if err := autotag.ImagePerformers(t.image, r.Image(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging image performers for %s: %v", t.image.Path, err) return fmt.Errorf("error tagging image performers for %s: %v", t.image.Path, err)
@@ -745,9 +743,9 @@ type autoTagGalleryTask struct {
cache *match.Cache cache *match.Cache
} }
func (t *autoTagGalleryTask) Start(wg *sync.WaitGroup) { func (t *autoTagGalleryTask) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
if t.performers { if t.performers {
if err := autotag.GalleryPerformers(t.gallery, r.Gallery(), r.Performer(), t.cache); err != nil { if err := autotag.GalleryPerformers(t.gallery, r.Gallery(), r.Performer(), t.cache); err != nil {
return fmt.Errorf("error tagging gallery performers for %s: %v", t.gallery.Path.String, err) return fmt.Errorf("error tagging gallery performers for %s: %v", t.gallery.Path.String, err)

8
internal/manager/task_clean.go
View File

@@ -29,7 +29,7 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) {
logger.Infof("Running in Dry Mode") logger.Infof("Running in Dry Mode")
} }
if err := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
total, err := j.getCount(r) total, err := j.getCount(r)
if err != nil { if err != nil {
return fmt.Errorf("error getting count: %w", err) return fmt.Errorf("error getting count: %w", err)
@@ -401,7 +401,7 @@ func (j *cleanJob) deleteScene(ctx context.Context, fileNamingAlgorithm models.H
Paths: GetInstance().Paths, Paths: GetInstance().Paths,
} }
var s *models.Scene var s *models.Scene
if err := j.txnManager.WithTxn(context.TODO(), func(repo models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error {
qb := repo.Scene() qb := repo.Scene()
var err error var err error
@@ -431,7 +431,7 @@ func (j *cleanJob) deleteScene(ctx context.Context, fileNamingAlgorithm models.H
func (j *cleanJob) deleteGallery(ctx context.Context, galleryID int) { func (j *cleanJob) deleteGallery(ctx context.Context, galleryID int) {
var g *models.Gallery var g *models.Gallery
if err := j.txnManager.WithTxn(context.TODO(), func(repo models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error {
qb := repo.Gallery() qb := repo.Gallery()
var err error var err error
@@ -459,7 +459,7 @@ func (j *cleanJob) deleteImage(ctx context.Context, imageID int) {
} }
var i *models.Image var i *models.Image
if err := j.txnManager.WithTxn(context.TODO(), func(repo models.Repository) error { if err := j.txnManager.WithTxn(ctx, func(repo models.Repository) error {
qb := repo.Image() qb := repo.Image()
var err error var err error

4
internal/manager/task_export.go
View File

@@ -96,7 +96,7 @@ func CreateExportTask(a models.HashAlgorithm, input models.ExportObjectsInput) *
} }
} }
func (t *ExportTask) Start(wg *sync.WaitGroup) { func (t *ExportTask) Start(ctx context.Context, wg *sync.WaitGroup) {
defer wg.Done() defer wg.Done()
// @manager.total = Scene.count + Gallery.count + Performer.count + Studio.count + Movie.count // @manager.total = Scene.count + Gallery.count + Performer.count + Studio.count + Movie.count
workerCount := runtime.GOMAXPROCS(0) // set worker count to number of cpus available workerCount := runtime.GOMAXPROCS(0) // set worker count to number of cpus available
@@ -129,7 +129,7 @@ func (t *ExportTask) Start(wg *sync.WaitGroup) {
paths.EnsureJSONDirs(t.baseDir) paths.EnsureJSONDirs(t.baseDir)
txnErr := t.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { txnErr := t.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
// include movie scenes and gallery images // include movie scenes and gallery images
if !t.full { if !t.full {
// only include movie scenes if includeDependencies is also set // only include movie scenes if includeDependencies is also set

8
internal/manager/task_generate.go
View File

@@ -75,7 +75,7 @@ func (j *GenerateJob) Execute(ctx context.Context, progress *job.Progress) {
if len(j.input.SceneIDs) > 0 { if len(j.input.SceneIDs) > 0 {
scenes, err = qb.FindMany(sceneIDs) scenes, err = qb.FindMany(sceneIDs)
for _, s := range scenes { for _, s := range scenes {
j.queueSceneJobs(s, queue, &totals) j.queueSceneJobs(ctx, s, queue, &totals)
} }
} }
@@ -165,7 +165,7 @@ func (j *GenerateJob) queueTasks(ctx context.Context, queue chan<- Task) totalsG
return context.Canceled return context.Canceled
} }
j.queueSceneJobs(ss, queue, &totals) j.queueSceneJobs(ctx, ss, queue, &totals)
} }
if len(scenes) != batchSize { if len(scenes) != batchSize {
@@ -185,7 +185,7 @@ func (j *GenerateJob) queueTasks(ctx context.Context, queue chan<- Task) totalsG
return totals return totals
} }
func (j *GenerateJob) queueSceneJobs(scene *models.Scene, queue chan<- Task, totals *totalsGenerate) { func (j *GenerateJob) queueSceneJobs(ctx context.Context, scene *models.Scene, queue chan<- Task, totals *totalsGenerate) {
if utils.IsTrue(j.input.Sprites) { if utils.IsTrue(j.input.Sprites) {
task := &GenerateSpriteTask{ task := &GenerateSpriteTask{
Scene: *scene, Scene: *scene,
@@ -243,7 +243,7 @@ func (j *GenerateJob) queueSceneJobs(scene *models.Scene, queue chan<- Task, tot
Screenshot: utils.IsTrue(j.input.MarkerScreenshots), Screenshot: utils.IsTrue(j.input.MarkerScreenshots),
} }
markers := task.markersNeeded() markers := task.markersNeeded(ctx)
if markers > 0 { if markers > 0 {
totals.markers += int64(markers) totals.markers += int64(markers)
totals.tasks++ totals.tasks++

4
internal/manager/task_generate_interactive_heatmap_speed.go
View File

@@ -47,7 +47,7 @@ func (t *GenerateInteractiveHeatmapSpeedTask) Start(ctx context.Context) {
var s *models.Scene var s *models.Scene
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
s, err = r.Scene().FindByPath(t.Scene.Path) s, err = r.Scene().FindByPath(t.Scene.Path)
return err return err
@@ -56,7 +56,7 @@ func (t *GenerateInteractiveHeatmapSpeedTask) Start(ctx context.Context) {
return return
} }
if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.TxnManager.WithTxn(ctx, func(r models.Repository) error {
qb := r.Scene() qb := r.Scene()
scenePartial := models.ScenePartial{ scenePartial := models.ScenePartial{
ID: s.ID, ID: s.ID,

12
internal/manager/task_generate_markers.go
View File

@@ -35,12 +35,12 @@ func (t *GenerateMarkersTask) GetDescription() string {
func (t *GenerateMarkersTask) Start(ctx context.Context) { func (t *GenerateMarkersTask) Start(ctx context.Context) {
if t.Scene != nil { if t.Scene != nil {
t.generateSceneMarkers() t.generateSceneMarkers(ctx)
} }
if t.Marker != nil { if t.Marker != nil {
var scene *models.Scene var scene *models.Scene
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
scene, err = r.Scene().Find(int(t.Marker.SceneID.Int64)) scene, err = r.Scene().Find(int(t.Marker.SceneID.Int64))
return err return err
@@ -65,9 +65,9 @@ func (t *GenerateMarkersTask) Start(ctx context.Context) {
} }
} }
func (t *GenerateMarkersTask) generateSceneMarkers() { func (t *GenerateMarkersTask) generateSceneMarkers(ctx context.Context) {
var sceneMarkers []*models.SceneMarker var sceneMarkers []*models.SceneMarker
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
sceneMarkers, err = r.SceneMarker().FindBySceneID(t.Scene.ID) sceneMarkers, err = r.SceneMarker().FindBySceneID(t.Scene.ID)
return err return err
@@ -167,10 +167,10 @@ func (t *GenerateMarkersTask) generateMarker(videoFile *ffmpeg.VideoFile, scene
} }
} }
func (t *GenerateMarkersTask) markersNeeded() int { func (t *GenerateMarkersTask) markersNeeded(ctx context.Context) int {
markers := 0 markers := 0
var sceneMarkers []*models.SceneMarker var sceneMarkers []*models.SceneMarker
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
sceneMarkers, err = r.SceneMarker().FindBySceneID(t.Scene.ID) sceneMarkers, err = r.SceneMarker().FindBySceneID(t.Scene.ID)
return err return err

2
internal/manager/task_generate_phash.go
View File

@@ -45,7 +45,7 @@ func (t *GeneratePhashTask) Start(ctx context.Context) {
return return
} }
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
qb := r.Scene() qb := r.Scene()
hashValue := sql.NullInt64{Int64: int64(*hash), Valid: true} hashValue := sql.NullInt64{Int64: int64(*hash), Valid: true}
scenePartial := models.ScenePartial{ scenePartial := models.ScenePartial{

2
internal/manager/task_generate_screenshot.go
View File

@@ -59,7 +59,7 @@ func (t *GenerateScreenshotTask) Start(ctx context.Context) {
return return
} }
if err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
qb := r.Scene() qb := r.Scene()
updatedTime := time.Now() updatedTime := time.Now()
updatedScene := models.ScenePartial{ updatedScene := models.ScenePartial{

14
internal/manager/task_scan.go
View File

@@ -97,7 +97,6 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
GenerateThumbnails: utils.IsTrue(input.ScanGenerateThumbnails), GenerateThumbnails: utils.IsTrue(input.ScanGenerateThumbnails),
progress: progress, progress: progress,
CaseSensitiveFs: f.caseSensitiveFs, CaseSensitiveFs: f.caseSensitiveFs,
ctx: ctx,
mutexManager: mutexManager, mutexManager: mutexManager,
} }
@@ -135,7 +134,7 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) {
UseFileMetadata: false, UseFileMetadata: false,
} }
go task.associateGallery(&wg) go task.associateGallery(ctx, &wg)
wg.Wait() wg.Wait()
} }
logger.Info("Finished gallery association") logger.Info("Finished gallery association")
@@ -187,7 +186,7 @@ func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, s
} }
total++ total++
if !j.doesPathExist(path) { if !j.doesPathExist(ctx, path) {
newFiles++ newFiles++
} }
@@ -212,14 +211,14 @@ func (j *ScanJob) queueFiles(ctx context.Context, paths []*models.StashConfig, s
return return
} }
func (j *ScanJob) doesPathExist(path string) bool { func (j *ScanJob) doesPathExist(ctx context.Context, path string) bool {
config := config.GetInstance() config := config.GetInstance()
vidExt := config.GetVideoExtensions() vidExt := config.GetVideoExtensions()
imgExt := config.GetImageExtensions() imgExt := config.GetImageExtensions()
gExt := config.GetGalleryExtensions() gExt := config.GetGalleryExtensions()
ret := false ret := false
txnErr := j.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { txnErr := j.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
switch { switch {
case fsutil.MatchExtension(path, gExt): case fsutil.MatchExtension(path, gExt):
g, _ := r.Gallery().FindByPath(path) g, _ := r.Gallery().FindByPath(path)
@@ -248,7 +247,6 @@ func (j *ScanJob) doesPathExist(path string) bool {
} }
type ScanTask struct { type ScanTask struct {
ctx context.Context
TxnManager models.TransactionManager TxnManager models.TransactionManager
file file.SourceFile file file.SourceFile
UseFileMetadata bool UseFileMetadata bool
@@ -275,9 +273,9 @@ func (t *ScanTask) Start(ctx context.Context) {
case isGallery(path): case isGallery(path):
t.scanGallery(ctx) t.scanGallery(ctx)
case isVideo(path): case isVideo(path):
s = t.scanScene() s = t.scanScene(ctx)
case isImage(path): case isImage(path):
t.scanImage() t.scanImage(ctx)
} }
}) })

21
internal/manager/task_scan_gallery.go
View File

@@ -42,7 +42,6 @@ func (t *ScanTask) scanGallery(ctx context.Context) {
Scanner: gallery.FileScanner(&file.FSHasher{}), Scanner: gallery.FileScanner(&file.FSHasher{}),
ImageExtensions: instance.Config.GetImageExtensions(), ImageExtensions: instance.Config.GetImageExtensions(),
StripFileExtension: t.StripFileExtension, StripFileExtension: t.StripFileExtension,
Ctx: t.ctx,
CaseSensitiveFs: t.CaseSensitiveFs, CaseSensitiveFs: t.CaseSensitiveFs,
TxnManager: t.TxnManager, TxnManager: t.TxnManager,
Paths: instance.Paths, Paths: instance.Paths,
@@ -52,7 +51,7 @@ func (t *ScanTask) scanGallery(ctx context.Context) {
var err error var err error
if g != nil { if g != nil {
g, scanImages, err = scanner.ScanExisting(g, t.file) g, scanImages, err = scanner.ScanExisting(ctx, g, t.file)
if err != nil { if err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
return return
@@ -61,7 +60,7 @@ func (t *ScanTask) scanGallery(ctx context.Context) {
// scan the zip files if the gallery has no images // scan the zip files if the gallery has no images
scanImages = scanImages || images == 0 scanImages = scanImages || images == 0
} else { } else {
g, scanImages, err = scanner.ScanNew(t.file) g, scanImages, err = scanner.ScanNew(ctx, t.file)
if err != nil { if err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
} }
@@ -69,18 +68,18 @@ func (t *ScanTask) scanGallery(ctx context.Context) {
if g != nil { if g != nil {
if scanImages { if scanImages {
t.scanZipImages(g) t.scanZipImages(ctx, g)
} else { } else {
// in case thumbnails have been deleted, regenerate them // in case thumbnails have been deleted, regenerate them
t.regenerateZipImages(g) t.regenerateZipImages(ctx, g)
} }
} }
} }
// associates a gallery to a scene with the same basename // associates a gallery to a scene with the same basename
func (t *ScanTask) associateGallery(wg *sizedwaitgroup.SizedWaitGroup) { func (t *ScanTask) associateGallery(ctx context.Context, wg *sizedwaitgroup.SizedWaitGroup) {
path := t.file.Path() path := t.file.Path()
if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.TxnManager.WithTxn(ctx, func(r models.Repository) error {
qb := r.Gallery() qb := r.Gallery()
sqb := r.Scene() sqb := r.Scene()
g, err := qb.FindByPath(path) g, err := qb.FindByPath(path)
@@ -133,7 +132,7 @@ func (t *ScanTask) associateGallery(wg *sizedwaitgroup.SizedWaitGroup) {
wg.Done() wg.Done()
} }
func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) { func (t *ScanTask) scanZipImages(ctx context.Context, zipGallery *models.Gallery) {
err := walkGalleryZip(zipGallery.Path.String, func(f *zip.File) error { err := walkGalleryZip(zipGallery.Path.String, func(f *zip.File) error {
// copy this task and change the filename // copy this task and change the filename
subTask := *t subTask := *t
@@ -143,7 +142,7 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) {
subTask.zipGallery = zipGallery subTask.zipGallery = zipGallery
// run the subtask and wait for it to complete // run the subtask and wait for it to complete
subTask.Start(context.TODO()) subTask.Start(ctx)
return nil return nil
}) })
if err != nil { if err != nil {
@@ -151,9 +150,9 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) {
} }
} }
func (t *ScanTask) regenerateZipImages(zipGallery *models.Gallery) { func (t *ScanTask) regenerateZipImages(ctx context.Context, zipGallery *models.Gallery) {
var images []*models.Image var images []*models.Image
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
iqb := r.Image() iqb := r.Image()
var err error var err error

15
internal/manager/task_scan_image.go
View File

@@ -18,11 +18,11 @@ import (
"github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/plugin"
) )
func (t *ScanTask) scanImage() { func (t *ScanTask) scanImage(ctx context.Context) {
var i *models.Image var i *models.Image
path := t.file.Path() path := t.file.Path()
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
i, err = r.Image().FindByPath(path) i, err = r.Image().FindByPath(path)
return err return err
@@ -34,7 +34,6 @@ func (t *ScanTask) scanImage() {
scanner := image.Scanner{ scanner := image.Scanner{
Scanner: image.FileScanner(&file.FSHasher{}), Scanner: image.FileScanner(&file.FSHasher{}),
StripFileExtension: t.StripFileExtension, StripFileExtension: t.StripFileExtension,
Ctx: t.ctx,
TxnManager: t.TxnManager, TxnManager: t.TxnManager,
Paths: GetInstance().Paths, Paths: GetInstance().Paths,
PluginCache: instance.PluginCache, PluginCache: instance.PluginCache,
@@ -43,13 +42,13 @@ func (t *ScanTask) scanImage() {
var err error var err error
if i != nil { if i != nil {
i, err = scanner.ScanExisting(i, t.file) i, err = scanner.ScanExisting(ctx, i, t.file)
if err != nil { if err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
return return
} }
} else { } else {
i, err = scanner.ScanNew(t.file) i, err = scanner.ScanNew(ctx, t.file)
if err != nil { if err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
return return
@@ -58,7 +57,7 @@ func (t *ScanTask) scanImage() {
if i != nil { if i != nil {
if t.zipGallery != nil { if t.zipGallery != nil {
// associate with gallery // associate with gallery
if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.TxnManager.WithTxn(ctx, func(r models.Repository) error {
return gallery.AddImage(r.Gallery(), t.zipGallery.ID, i.ID) return gallery.AddImage(r.Gallery(), t.zipGallery.ID, i.ID)
}); err != nil { }); err != nil {
logger.Error(err.Error()) logger.Error(err.Error())
@@ -69,7 +68,7 @@ func (t *ScanTask) scanImage() {
logger.Infof("Associating image %s with folder gallery", i.Path) logger.Infof("Associating image %s with folder gallery", i.Path)
var galleryID int var galleryID int
var isNewGallery bool var isNewGallery bool
if err := t.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := t.TxnManager.WithTxn(ctx, func(r models.Repository) error {
var err error var err error
galleryID, isNewGallery, err = t.associateImageWithFolderGallery(i.ID, r.Gallery()) galleryID, isNewGallery, err = t.associateImageWithFolderGallery(i.ID, r.Gallery())
return err return err
@@ -79,7 +78,7 @@ func (t *ScanTask) scanImage() {
} }
if isNewGallery { if isNewGallery {
GetInstance().PluginCache.ExecutePostHooks(t.ctx, galleryID, plugin.GalleryCreatePost, nil, nil) GetInstance().PluginCache.ExecutePostHooks(ctx, galleryID, plugin.GalleryCreatePost, nil, nil)
} }
} }
} }

9
internal/manager/task_scan_scene.go
View File

@@ -9,7 +9,7 @@ import (
"github.com/stashapp/stash/pkg/scene" "github.com/stashapp/stash/pkg/scene"
) )
func (t *ScanTask) scanScene() *models.Scene { func (t *ScanTask) scanScene(ctx context.Context) *models.Scene {
logError := func(err error) *models.Scene { logError := func(err error) *models.Scene {
logger.Error(err.Error()) logger.Error(err.Error())
return nil return nil
@@ -18,7 +18,7 @@ func (t *ScanTask) scanScene() *models.Scene {
var retScene *models.Scene var retScene *models.Scene
var s *models.Scene var s *models.Scene
if err := t.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := t.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
s, err = r.Scene().FindByPath(t.file.Path()) s, err = r.Scene().FindByPath(t.file.Path())
return err return err
@@ -31,7 +31,6 @@ func (t *ScanTask) scanScene() *models.Scene {
Scanner: scene.FileScanner(&file.FSHasher{}, t.fileNamingAlgorithm, t.calculateMD5), Scanner: scene.FileScanner(&file.FSHasher{}, t.fileNamingAlgorithm, t.calculateMD5),
StripFileExtension: t.StripFileExtension, StripFileExtension: t.StripFileExtension,
FileNamingAlgorithm: t.fileNamingAlgorithm, FileNamingAlgorithm: t.fileNamingAlgorithm,
Ctx: t.ctx,
TxnManager: t.TxnManager, TxnManager: t.TxnManager,
Paths: GetInstance().Paths, Paths: GetInstance().Paths,
Screenshotter: &instance.FFMPEG, Screenshotter: &instance.FFMPEG,
@@ -42,7 +41,7 @@ func (t *ScanTask) scanScene() *models.Scene {
} }
if s != nil { if s != nil {
if err := scanner.ScanExisting(s, t.file); err != nil { if err := scanner.ScanExisting(ctx, s, t.file); err != nil {
return logError(err) return logError(err)
} }
@@ -50,7 +49,7 @@ func (t *ScanTask) scanScene() *models.Scene {
} }
var err error var err error
retScene, err = scanner.ScanNew(t.file) retScene, err = scanner.ScanNew(ctx, t.file)
if err != nil { if err != nil {
return logError(err) return logError(err)
} }

8
internal/manager/task_stash_box_tag.go
View File

@@ -22,8 +22,8 @@ type StashBoxPerformerTagTask struct {
excluded_fields []string excluded_fields []string
} }
func (t *StashBoxPerformerTagTask) Start() { func (t *StashBoxPerformerTagTask) Start(ctx context.Context) {
t.stashBoxPerformerTag(context.TODO()) t.stashBoxPerformerTag(ctx)
} }
func (t *StashBoxPerformerTagTask) Description() string { func (t *StashBoxPerformerTagTask) Description() string {
@@ -156,7 +156,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag(ctx context.Context) {
partial.URL = &value partial.URL = &value
} }
txnErr := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { txnErr := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
_, err := r.Performer().Update(partial) _, err := r.Performer().Update(partial)
if !t.refresh { if !t.refresh {
@@ -218,7 +218,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag(ctx context.Context) {
URL: getNullString(performer.URL), URL: getNullString(performer.URL),
UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
} }
err := t.txnManager.WithTxn(context.TODO(), func(r models.Repository) error { err := t.txnManager.WithTxn(ctx, func(r models.Repository) error {
createdPerformer, err := r.Performer().Create(newPerformer) createdPerformer, err := r.Performer().Create(newPerformer)
if err != nil { if err != nil {
return err return err

15
pkg/gallery/scan.go
View File

@@ -24,7 +24,6 @@ type Scanner struct {
ImageExtensions []string ImageExtensions []string
StripFileExtension bool StripFileExtension bool
Ctx context.Context
CaseSensitiveFs bool CaseSensitiveFs bool
TxnManager models.TransactionManager TxnManager models.TransactionManager
Paths *paths.Paths Paths *paths.Paths
@@ -39,7 +38,7 @@ func FileScanner(hasher file.Hasher) file.Scanner {
} }
} }
func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) { func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) {
scanned, err := scanner.Scanner.ScanExisting(existing, file) scanned, err := scanner.Scanner.ScanExisting(existing, file)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
@@ -76,7 +75,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
done := make(chan struct{}) done := make(chan struct{})
scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done)
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
// free the mutex once transaction is complete // free the mutex once transaction is complete
defer close(done) defer close(done)
@@ -94,13 +93,13 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
return nil, false, err return nil, false, err
} }
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retGallery.ID, plugin.GalleryUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, retGallery.ID, plugin.GalleryUpdatePost, nil, nil)
} }
return return
} }
func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) { func (scanner *Scanner) ScanNew(ctx context.Context, file file.SourceFile) (retGallery *models.Gallery, scanImages bool, err error) {
scanned, err := scanner.Scanner.ScanNew(file) scanned, err := scanner.Scanner.ScanNew(file)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
@@ -117,7 +116,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler
scanner.MutexManager.Claim(mutexType, checksum, done) scanner.MutexManager.Claim(mutexType, checksum, done)
defer close(done) defer close(done)
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
qb := r.Gallery() qb := r.Gallery()
g, _ = qb.FindByChecksum(checksum) g, _ = qb.FindByChecksum(checksum)
@@ -183,9 +182,9 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retGallery *models.Galler
} }
if isNewGallery { if isNewGallery {
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, g.ID, plugin.GalleryCreatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, g.ID, plugin.GalleryCreatePost, nil, nil)
} else if isUpdatedGallery { } else if isUpdatedGallery {
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, g.ID, plugin.GalleryUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, g.ID, plugin.GalleryUpdatePost, nil, nil)
} }
scanImages = isNewGallery scanImages = isNewGallery

19
pkg/image/scan.go
View File

@@ -22,7 +22,6 @@ type Scanner struct {
StripFileExtension bool StripFileExtension bool
Ctx context.Context
CaseSensitiveFs bool CaseSensitiveFs bool
TxnManager models.TransactionManager TxnManager models.TransactionManager
Paths *paths.Paths Paths *paths.Paths
@@ -37,7 +36,7 @@ func FileScanner(hasher file.Hasher) file.Scanner {
} }
} }
func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFile) (retImage *models.Image, err error) { func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (retImage *models.Image, err error) {
scanned, err := scanner.Scanner.ScanExisting(existing, file) scanned, err := scanner.Scanner.ScanExisting(existing, file)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -72,7 +71,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
done := make(chan struct{}) done := make(chan struct{})
scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done)
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
// free the mutex once transaction is complete // free the mutex once transaction is complete
defer close(done) defer close(done)
var err error var err error
@@ -100,13 +99,13 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
} }
} }
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retImage.ID, plugin.ImageUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, retImage.ID, plugin.ImageUpdatePost, nil, nil)
} }
return return
} }
func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err error) { func (scanner *Scanner) ScanNew(ctx context.Context, f file.SourceFile) (retImage *models.Image, err error) {
scanned, err := scanner.Scanner.ScanNew(f) scanned, err := scanner.Scanner.ScanNew(f)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -122,7 +121,7 @@ func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err
// check for image by checksum // check for image by checksum
var existingImage *models.Image var existingImage *models.Image
if err := scanner.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := scanner.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
var err error var err error
existingImage, err = r.Image().FindByChecksum(checksum) existingImage, err = r.Image().FindByChecksum(checksum)
return err return err
@@ -152,14 +151,14 @@ func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err
Path: &path, Path: &path,
} }
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
retImage, err = r.Image().Update(imagePartial) retImage, err = r.Image().Update(imagePartial)
return err return err
}); err != nil { }); err != nil {
return nil, err return nil, err
} }
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, existingImage.ID, plugin.ImageUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, existingImage.ID, plugin.ImageUpdatePost, nil, nil)
} }
} else { } else {
logger.Infof("%s doesn't exist. Creating new item...", pathDisplayName) logger.Infof("%s doesn't exist. Creating new item...", pathDisplayName)
@@ -177,7 +176,7 @@ func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err
return nil, err return nil, err
} }
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
var err error var err error
retImage, err = r.Image().Create(newImage) retImage, err = r.Image().Create(newImage)
return err return err
@@ -185,7 +184,7 @@ func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err
return nil, err return nil, err
} }
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retImage.ID, plugin.ImageCreatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, retImage.ID, plugin.ImageCreatePost, nil, nil)
} }
return return

10
pkg/job/manager.go
View File

@@ -92,7 +92,7 @@ func (m *Manager) Start(ctx context.Context, description string, e JobExec) int
m.queue = append(m.queue, &j) m.queue = append(m.queue, &j)
m.dispatch(&j) m.dispatch(ctx, &j)
return j.ID return j.ID
} }
@@ -145,7 +145,7 @@ func (m *Manager) dispatcher() {
} }
} }
done := m.dispatch(j) done := m.dispatch(j.outerCtx, j)
// unlock the mutex and wait for the job to finish // unlock the mutex and wait for the job to finish
m.mutex.Unlock() m.mutex.Unlock()
@@ -169,15 +169,13 @@ func (m *Manager) newProgress(j *Job) *Progress {
} }
} }
func (m *Manager) dispatch(j *Job) (done chan struct{}) { func (m *Manager) dispatch(ctx context.Context, j *Job) (done chan struct{}) {
// assumes lock held // assumes lock held
t := time.Now() t := time.Now()
j.StartTime = &t j.StartTime = &t
j.Status = StatusRunning j.Status = StatusRunning
ctx, cancelFunc := context.WithCancel(valueOnlyContext{ ctx, cancelFunc := context.WithCancel(valueOnlyContext{ctx})
j.outerCtx,
})
j.cancelFunc = cancelFunc j.cancelFunc = cancelFunc
done = make(chan struct{}) done = make(chan struct{})

5
pkg/plugin/js/log.go
View File

@@ -15,7 +15,10 @@ func argToString(call otto.FunctionCall) string {
arg := call.Argument(0) arg := call.Argument(0)
if arg.IsObject() { if arg.IsObject() {
o, _ := arg.Export() o, _ := arg.Export()
data, _ := json.Marshal(o) data, err := json.Marshal(o)
if err != nil {
logger.Warnf("Couldn't json encode object")
}
return string(data) return string(data)
} }

5
pkg/plugin/raw.go
View File

@@ -71,7 +71,10 @@ func (t *rawPluginTask) Start() error {
go func() { go func() {
defer stdin.Close() defer stdin.Close()
inBytes, _ := json.Marshal(t.input) inBytes, err := json.Marshal(t.input)
if err != nil {
logger.Warnf("error marshalling raw command input")
}
if k, err := io.WriteString(stdin, string(inBytes)); err != nil { if k, err := io.WriteString(stdin, string(inBytes)); err != nil {
logger.Warnf("error writing input to plugins stdin (wrote %v bytes out of %v): %v", k, len(string(inBytes)), err) logger.Warnf("error writing input to plugins stdin (wrote %v bytes out of %v): %v", k, len(string(inBytes)), err)
} }

19
pkg/scene/scan.go
View File

@@ -32,7 +32,6 @@ type Scanner struct {
UseFileMetadata bool UseFileMetadata bool
FileNamingAlgorithm models.HashAlgorithm FileNamingAlgorithm models.HashAlgorithm
Ctx context.Context
CaseSensitiveFs bool CaseSensitiveFs bool
TxnManager models.TransactionManager TxnManager models.TransactionManager
Paths *paths.Paths Paths *paths.Paths
@@ -50,7 +49,7 @@ func FileScanner(hasher file.Hasher, fileNamingAlgorithm models.HashAlgorithm, c
} }
} }
func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFile) (err error) { func (scanner *Scanner) ScanExisting(ctx context.Context, existing file.FileBased, file file.SourceFile) (err error) {
scanned, err := scanner.Scanner.ScanExisting(existing, file) scanned, err := scanner.Scanner.ScanExisting(existing, file)
if err != nil { if err != nil {
return err return err
@@ -110,7 +109,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done) scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done)
} }
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
defer close(done) defer close(done)
qb := r.Scene() qb := r.Scene()
@@ -144,7 +143,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
MigrateHash(scanner.Paths, oldHash, newHash) MigrateHash(scanner.Paths, oldHash, newHash)
} }
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, s.ID, plugin.SceneUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil)
} }
// We already have this item in the database // We already have this item in the database
@@ -154,7 +153,7 @@ func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFi
return nil return nil
} }
func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, err error) { func (scanner *Scanner) ScanNew(ctx context.Context, file file.SourceFile) (retScene *models.Scene, err error) {
scanned, err := scanner.Scanner.ScanNew(file) scanned, err := scanner.Scanner.ScanNew(file)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -178,7 +177,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e
// check for scene by checksum and oshash - MD5 should be // check for scene by checksum and oshash - MD5 should be
// redundant, but check both // redundant, but check both
var s *models.Scene var s *models.Scene
if err := scanner.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { if err := scanner.TxnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
qb := r.Scene() qb := r.Scene()
if checksum != "" { if checksum != "" {
s, _ = qb.FindByChecksum(checksum) s, _ = qb.FindByChecksum(checksum)
@@ -220,7 +219,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e
Path: &path, Path: &path,
Interactive: &interactive, Interactive: &interactive,
} }
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
_, err := r.Scene().Update(scenePartial) _, err := r.Scene().Update(scenePartial)
return err return err
}); err != nil { }); err != nil {
@@ -228,7 +227,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e
} }
scanner.makeScreenshots(path, nil, sceneHash) scanner.makeScreenshots(path, nil, sceneHash)
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, s.ID, plugin.SceneUpdatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil)
} }
} else { } else {
logger.Infof("%s doesn't exist. Creating new item...", path) logger.Infof("%s doesn't exist. Creating new item...", path)
@@ -265,7 +264,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e
_ = newScene.Date.Scan(videoFile.CreationTime) _ = newScene.Date.Scan(videoFile.CreationTime)
} }
if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error { if err := scanner.TxnManager.WithTxn(ctx, func(r models.Repository) error {
var err error var err error
retScene, err = r.Scene().Create(newScene) retScene, err = r.Scene().Create(newScene)
return err return err
@@ -274,7 +273,7 @@ func (scanner *Scanner) ScanNew(file file.SourceFile) (retScene *models.Scene, e
} }
scanner.makeScreenshots(path, videoFile, sceneHash) scanner.makeScreenshots(path, videoFile, sceneHash)
scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retScene.ID, plugin.SceneCreatePost, nil, nil) scanner.PluginCache.ExecutePostHooks(ctx, retScene.ID, plugin.SceneCreatePost, nil, nil)
} }
return retScene, nil return retScene, nil

20
pkg/scraper/script.go
View File

@@ -32,7 +32,7 @@ func newScriptScraper(scraper scraperTypeConfig, config config, globalConfig Glo
} }
} }
func (s *scriptScraper) runScraperScript(inString string, out interface{}) error { func (s *scriptScraper) runScraperScript(ctx context.Context, inString string, out interface{}) error {
command := s.scraper.Script command := s.scraper.Script
var cmd *exec.Cmd var cmd *exec.Cmd
@@ -46,7 +46,7 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error
} }
if p != nil { if p != nil {
cmd = p.Command(context.TODO(), command[1:]) cmd = p.Command(ctx, command[1:])
} }
// if could not find python, just use the command args as-is // if could not find python, just use the command args as-is
@@ -133,7 +133,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty models
switch ty { switch ty {
case models.ScrapeContentTypePerformer: case models.ScrapeContentTypePerformer:
var performers []models.ScrapedPerformer var performers []models.ScrapedPerformer
err = s.runScraperScript(input, &performers) err = s.runScraperScript(ctx, input, &performers)
if err == nil { if err == nil {
for _, p := range performers { for _, p := range performers {
v := p v := p
@@ -142,7 +142,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty models
} }
case models.ScrapeContentTypeScene: case models.ScrapeContentTypeScene:
var scenes []models.ScrapedScene var scenes []models.ScrapedScene
err = s.runScraperScript(input, &scenes) err = s.runScraperScript(ctx, input, &scenes)
if err == nil { if err == nil {
for _, s := range scenes { for _, s := range scenes {
v := s v := s
@@ -187,19 +187,19 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty models.Scra
switch ty { switch ty {
case models.ScrapeContentTypePerformer: case models.ScrapeContentTypePerformer:
var performer *models.ScrapedPerformer var performer *models.ScrapedPerformer
err := s.runScraperScript(input, &performer) err := s.runScraperScript(ctx, input, &performer)
return performer, err return performer, err
case models.ScrapeContentTypeGallery: case models.ScrapeContentTypeGallery:
var gallery *models.ScrapedGallery var gallery *models.ScrapedGallery
err := s.runScraperScript(input, &gallery) err := s.runScraperScript(ctx, input, &gallery)
return gallery, err return gallery, err
case models.ScrapeContentTypeScene: case models.ScrapeContentTypeScene:
var scene *models.ScrapedScene var scene *models.ScrapedScene
err := s.runScraperScript(input, &scene) err := s.runScraperScript(ctx, input, &scene)
return scene, err return scene, err
case models.ScrapeContentTypeMovie: case models.ScrapeContentTypeMovie:
var movie *models.ScrapedMovie var movie *models.ScrapedMovie
err := s.runScraperScript(input, &movie) err := s.runScraperScript(ctx, input, &movie)
return movie, err return movie, err
} }
@@ -215,7 +215,7 @@ func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sc
var ret *models.ScrapedScene var ret *models.ScrapedScene
err = s.runScraperScript(string(inString), &ret) err = s.runScraperScript(ctx, string(inString), &ret)
return ret, err return ret, err
} }
@@ -229,7 +229,7 @@ func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mod
var ret *models.ScrapedGallery var ret *models.ScrapedGallery
err = s.runScraperScript(string(inString), &ret) err = s.runScraperScript(ctx, string(inString), &ret)
return ret, err return ret, err
} }