Refactor autotag to use individual transactions (#3106)

* Add id filtering to scenes, images, and galleries
* Perform tagging in batches
* One transaction per object tagged

graphql/schema/types/filters.graphql

@@ -123,6 +123,7 @@ input SceneFilterType {
   OR: SceneFilterType
   NOT: SceneFilterType
 
+  id: IntCriterionInput
   title: StringCriterionInput
   code: StringCriterionInput
   details: StringCriterionInput
@@ -238,6 +239,7 @@ input GalleryFilterType {
   OR: GalleryFilterType
   NOT: GalleryFilterType
 
+  id: IntCriterionInput
   title: StringCriterionInput
   details: StringCriterionInput
 
@@ -334,6 +336,8 @@ input ImageFilterType {
 
   title: StringCriterionInput
 
+  """ Filter by image id"""
+  id: IntCriterionInput
   """Filter by file checksum"""
   checksum: StringCriterionInput
   """Filter by path"""

internal/autotag/integration_test.go

@@ -479,6 +479,10 @@ func withTxn(f func(ctx context.Context) error) error {
 	return txn.WithTxn(context.TODO(), db, f)
 }
 
+func withDB(f func(ctx context.Context) error) error {
+	return txn.WithDatabase(context.TODO(), db, f)
+}
+
 func populateDB() error {
 	if err := withTxn(func(ctx context.Context) error {
 		err := createPerformer(ctx, r.Performer)
@@ -538,9 +542,13 @@ func TestParsePerformerScenes(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, p := range performers {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
-			return PerformerScenes(ctx, p, nil, r.Scene, nil)
+			return tagger.PerformerScenes(ctx, p, nil, r.Scene)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -585,14 +593,18 @@ func TestParseStudioScenes(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range studios {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Studio.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return StudioScenes(ctx, s, nil, aliases, r.Scene, nil)
+			return tagger.StudioScenes(ctx, s, nil, aliases, r.Scene)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -641,14 +653,18 @@ func TestParseTagScenes(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range tags {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Tag.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return TagScenes(ctx, s, nil, aliases, r.Scene, nil)
+			return tagger.TagScenes(ctx, s, nil, aliases, r.Scene)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -693,9 +709,13 @@ func TestParsePerformerImages(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, p := range performers {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
-			return PerformerImages(ctx, p, nil, r.Image, nil)
+			return tagger.PerformerImages(ctx, p, nil, r.Image)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -741,14 +761,18 @@ func TestParseStudioImages(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range studios {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Studio.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return StudioImages(ctx, s, nil, aliases, r.Image, nil)
+			return tagger.StudioImages(ctx, s, nil, aliases, r.Image)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -797,14 +821,18 @@ func TestParseTagImages(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range tags {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Tag.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return TagImages(ctx, s, nil, aliases, r.Image, nil)
+			return tagger.TagImages(ctx, s, nil, aliases, r.Image)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -850,9 +878,13 @@ func TestParsePerformerGalleries(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, p := range performers {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
-			return PerformerGalleries(ctx, p, nil, r.Gallery, nil)
+			return tagger.PerformerGalleries(ctx, p, nil, r.Gallery)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -898,14 +930,18 @@ func TestParseStudioGalleries(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range studios {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Studio.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return StudioGalleries(ctx, s, nil, aliases, r.Gallery, nil)
+			return tagger.StudioGalleries(ctx, s, nil, aliases, r.Gallery)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}
@@ -954,14 +990,18 @@ func TestParseTagGalleries(t *testing.T) {
 		return
 	}
 
+	tagger := Tagger{
+		TxnManager: db,
+	}
+
 	for _, s := range tags {
-		if err := withTxn(func(ctx context.Context) error {
+		if err := withDB(func(ctx context.Context) error {
 			aliases, err := r.Tag.GetAliases(ctx, s.ID)
 			if err != nil {
 				return err
 			}
 
-			return TagGalleries(ctx, s, nil, aliases, r.Gallery, nil)
+			return tagger.TagGalleries(ctx, s, nil, aliases, r.Gallery)
 		}); err != nil {
 			t.Errorf("Error auto-tagging performers: %s", err)
 		}

internal/autotag/performer.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 type SceneQueryPerformerUpdater interface {
@@ -39,8 +40,8 @@ func getPerformerTagger(p *models.Performer, cache *match.Cache) tagger {
 }
 
 // PerformerScenes searches for scenes whose path matches the provided performer name and tags the scene with the performer.
-func PerformerScenes(ctx context.Context, p *models.Performer, paths []string, rw SceneQueryPerformerUpdater, cache *match.Cache) error {
+func (tagger *Tagger) PerformerScenes(ctx context.Context, p *models.Performer, paths []string, rw SceneQueryPerformerUpdater) error {
-	t := getPerformerTagger(p, cache)
+	t := getPerformerTagger(p, tagger.Cache)
 
 	return t.tagScenes(ctx, paths, rw, func(o *models.Scene) (bool, error) {
 		if err := o.LoadPerformerIDs(ctx, rw); err != nil {
@@ -52,7 +53,9 @@ func PerformerScenes(ctx context.Context, p *models.Performer, paths []string, r
 			return false, nil
 		}
 
-		if err := scene.AddPerformer(ctx, rw, o, p.ID); err != nil {
+		if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+			return scene.AddPerformer(ctx, rw, o, p.ID)
+		}); err != nil {
 			return false, err
 		}
 
@@ -61,8 +64,8 @@ func PerformerScenes(ctx context.Context, p *models.Performer, paths []string, r
 }
 
 // PerformerImages searches for images whose path matches the provided performer name and tags the image with the performer.
-func PerformerImages(ctx context.Context, p *models.Performer, paths []string, rw ImageQueryPerformerUpdater, cache *match.Cache) error {
+func (tagger *Tagger) PerformerImages(ctx context.Context, p *models.Performer, paths []string, rw ImageQueryPerformerUpdater) error {
-	t := getPerformerTagger(p, cache)
+	t := getPerformerTagger(p, tagger.Cache)
 
 	return t.tagImages(ctx, paths, rw, func(o *models.Image) (bool, error) {
 		if err := o.LoadPerformerIDs(ctx, rw); err != nil {
@@ -74,7 +77,9 @@ func PerformerImages(ctx context.Context, p *models.Performer, paths []string, r
 			return false, nil
 		}
 
-		if err := image.AddPerformer(ctx, rw, o, p.ID); err != nil {
+		if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+			return image.AddPerformer(ctx, rw, o, p.ID)
+		}); err != nil {
 			return false, err
 		}
 
@@ -83,8 +88,8 @@ func PerformerImages(ctx context.Context, p *models.Performer, paths []string, r
 }
 
 // PerformerGalleries searches for galleries whose path matches the provided performer name and tags the gallery with the performer.
-func PerformerGalleries(ctx context.Context, p *models.Performer, paths []string, rw GalleryQueryPerformerUpdater, cache *match.Cache) error {
+func (tagger *Tagger) PerformerGalleries(ctx context.Context, p *models.Performer, paths []string, rw GalleryQueryPerformerUpdater) error {
-	t := getPerformerTagger(p, cache)
+	t := getPerformerTagger(p, tagger.Cache)
 
 	return t.tagGalleries(ctx, paths, rw, func(o *models.Gallery) (bool, error) {
 		if err := o.LoadPerformerIDs(ctx, rw); err != nil {
@@ -96,7 +101,9 @@ func PerformerGalleries(ctx context.Context, p *models.Performer, paths []string
 			return false, nil
 		}
 
-		if err := gallery.AddPerformer(ctx, rw, o, p.ID); err != nil {
+		if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+			return gallery.AddPerformer(ctx, rw, o, p.ID)
+		}); err != nil {
 			return false, err
 		}
 

internal/autotag/performer_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/models/mocks"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 func TestPerformerScenes(t *testing.T) {
@@ -64,7 +65,9 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedSceneFilter := &models.SceneFilterType{
 		Organized: &organized,
@@ -76,14 +79,16 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
-	mockSceneReader.On("Query", testCtx, scene.QueryOptions(expectedSceneFilter, expectedFindFilter, false)).
+	mockSceneReader.On("Query", mock.Anything, scene.QueryOptions(expectedSceneFilter, expectedFindFilter, false)).
 		Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 
 	for i := range matchingPaths {
 		sceneID := i + 1
-		mockSceneReader.On("UpdatePartial", testCtx, sceneID, models.ScenePartial{
+		mockSceneReader.On("UpdatePartial", mock.Anything, sceneID, models.ScenePartial{
 			PerformerIDs: &models.UpdateIDs{
 				IDs:  []int{performerID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -91,7 +96,11 @@ func testPerformerScenes(t *testing.T, performerName, expectedRegex string) {
 		}).Return(nil, nil).Once()
 	}
 
-	err := PerformerScenes(testCtx, &performer, nil, mockSceneReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.PerformerScenes(testCtx, &performer, nil, mockSceneReader)
 
 	assert := assert.New(t)
 
@@ -144,7 +153,9 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedImageFilter := &models.ImageFilterType{
 		Organized: &organized,
@@ -156,14 +167,16 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
-	mockImageReader.On("Query", testCtx, image.QueryOptions(expectedImageFilter, expectedFindFilter, false)).
+	mockImageReader.On("Query", mock.Anything, image.QueryOptions(expectedImageFilter, expectedFindFilter, false)).
 		Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 
 	for i := range matchingPaths {
 		imageID := i + 1
-		mockImageReader.On("UpdatePartial", testCtx, imageID, models.ImagePartial{
+		mockImageReader.On("UpdatePartial", mock.Anything, imageID, models.ImagePartial{
 			PerformerIDs: &models.UpdateIDs{
 				IDs:  []int{performerID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -171,7 +184,11 @@ func testPerformerImages(t *testing.T, performerName, expectedRegex string) {
 		}).Return(nil, nil).Once()
 	}
 
-	err := PerformerImages(testCtx, &performer, nil, mockImageReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.PerformerImages(testCtx, &performer, nil, mockImageReader)
 
 	assert := assert.New(t)
 
@@ -225,7 +242,9 @@ func testPerformerGalleries(t *testing.T, performerName, expectedRegex string) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedGalleryFilter := &models.GalleryFilterType{
 		Organized: &organized,
@@ -237,13 +256,15 @@ func testPerformerGalleries(t *testing.T, performerName, expectedRegex string) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
-	mockGalleryReader.On("Query", testCtx, expectedGalleryFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
+	mockGalleryReader.On("Query", mock.Anything, expectedGalleryFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
 
 	for i := range matchingPaths {
 		galleryID := i + 1
-		mockGalleryReader.On("UpdatePartial", testCtx, galleryID, models.GalleryPartial{
+		mockGalleryReader.On("UpdatePartial", mock.Anything, galleryID, models.GalleryPartial{
 			PerformerIDs: &models.UpdateIDs{
 				IDs:  []int{performerID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -251,7 +272,11 @@ func testPerformerGalleries(t *testing.T, performerName, expectedRegex string) {
 		}).Return(nil, nil).Once()
 	}
 
-	err := PerformerGalleries(testCtx, &performer, nil, mockGalleryReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.PerformerGalleries(testCtx, &performer, nil, mockGalleryReader)
 
 	assert := assert.New(t)
 

internal/autotag/studio.go

@@ -8,8 +8,12 @@ import (
 	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
+// the following functions aren't used in Tagger because they assume
+// use within a transaction
+
 func addSceneStudio(ctx context.Context, sceneWriter scene.PartialUpdater, o *models.Scene, studioID int) (bool, error) {
 	// don't set if already set
 	if o.StudioID != nil {
@@ -86,12 +90,28 @@ type SceneFinderUpdater interface {
 }
 
 // StudioScenes searches for scenes whose path matches the provided studio name and tags the scene with the studio, if studio is not already set on the scene.
-func StudioScenes(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw SceneFinderUpdater, cache *match.Cache) error {
+func (tagger *Tagger) StudioScenes(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw SceneFinderUpdater) error {
-	t := getStudioTagger(p, aliases, cache)
+	t := getStudioTagger(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagScenes(ctx, paths, rw, func(o *models.Scene) (bool, error) {
-			return addSceneStudio(ctx, rw, o, p.ID)
+			// don't set if already set
+			if o.StudioID != nil {
+				return false, nil
+			}
+
+			// set the studio id
+			scenePartial := models.ScenePartial{
+				StudioID: models.NewOptionalInt(p.ID),
+			}
+
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				_, err := rw.UpdatePartial(ctx, o.ID, scenePartial)
+				return err
+			}); err != nil {
+				return false, err
+			}
+			return true, nil
 		}); err != nil {
 			return err
 		}
@@ -107,12 +127,28 @@ type ImageFinderUpdater interface {
 }
 
 // StudioImages searches for images whose path matches the provided studio name and tags the image with the studio, if studio is not already set on the image.
-func StudioImages(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw ImageFinderUpdater, cache *match.Cache) error {
+func (tagger *Tagger) StudioImages(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw ImageFinderUpdater) error {
-	t := getStudioTagger(p, aliases, cache)
+	t := getStudioTagger(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagImages(ctx, paths, rw, func(i *models.Image) (bool, error) {
-			return addImageStudio(ctx, rw, i, p.ID)
+			// don't set if already set
+			if i.StudioID != nil {
+				return false, nil
+			}
+
+			// set the studio id
+			imagePartial := models.ImagePartial{
+				StudioID: models.NewOptionalInt(p.ID),
+			}
+
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				_, err := rw.UpdatePartial(ctx, i.ID, imagePartial)
+				return err
+			}); err != nil {
+				return false, err
+			}
+			return true, nil
 		}); err != nil {
 			return err
 		}
@@ -128,12 +164,28 @@ type GalleryFinderUpdater interface {
 }
 
 // StudioGalleries searches for galleries whose path matches the provided studio name and tags the gallery with the studio, if studio is not already set on the gallery.
-func StudioGalleries(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw GalleryFinderUpdater, cache *match.Cache) error {
+func (tagger *Tagger) StudioGalleries(ctx context.Context, p *models.Studio, paths []string, aliases []string, rw GalleryFinderUpdater) error {
-	t := getStudioTagger(p, aliases, cache)
+	t := getStudioTagger(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagGalleries(ctx, paths, rw, func(o *models.Gallery) (bool, error) {
-			return addGalleryStudio(ctx, rw, o, p.ID)
+			// don't set if already set
+			if o.StudioID != nil {
+				return false, nil
+			}
+
+			// set the studio id
+			galleryPartial := models.GalleryPartial{
+				StudioID: models.NewOptionalInt(p.ID),
+			}
+
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				_, err := rw.UpdatePartial(ctx, o.ID, galleryPartial)
+				return err
+			}); err != nil {
+				return false, err
+			}
+			return true, nil
 		}); err != nil {
 			return err
 		}

internal/autotag/studio_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/models/mocks"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 type testStudioCase struct {
@@ -110,7 +111,9 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedSceneFilter := &models.SceneFilterType{
 		Organized: &organized,
@@ -122,6 +125,8 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
@@ -140,19 +145,23 @@ func testStudioScenes(t *testing.T, tc testStudioCase) {
 			},
 		}
 
-		mockSceneReader.On("Query", testCtx, scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+		mockSceneReader.On("Query", mock.Anything, scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
 			Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		sceneID := i + 1
 		expectedStudioID := studioID
-		mockSceneReader.On("UpdatePartial", testCtx, sceneID, models.ScenePartial{
+		mockSceneReader.On("UpdatePartial", mock.Anything, sceneID, models.ScenePartial{
 			StudioID: models.NewOptionalInt(expectedStudioID),
 		}).Return(nil, nil).Once()
 	}
 
-	err := StudioScenes(testCtx, &studio, nil, aliases, mockSceneReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.StudioScenes(testCtx, &studio, nil, aliases, mockSceneReader)
 
 	assert := assert.New(t)
 
@@ -201,7 +210,9 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedImageFilter := &models.ImageFilterType{
 		Organized: &organized,
@@ -213,10 +224,12 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockImageReader.On("Query", testCtx, image.QueryOptions(expectedImageFilter, expectedFindFilter, false))
+	onNameQuery := mockImageReader.On("Query", mock.Anything, image.QueryOptions(expectedImageFilter, expectedFindFilter, false))
 	if aliasName == "" {
 		onNameQuery.Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	} else {
@@ -230,19 +243,23 @@ func testStudioImages(t *testing.T, tc testStudioCase) {
 			},
 		}
 
-		mockImageReader.On("Query", testCtx, image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+		mockImageReader.On("Query", mock.Anything, image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
 			Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		imageID := i + 1
 		expectedStudioID := studioID
-		mockImageReader.On("UpdatePartial", testCtx, imageID, models.ImagePartial{
+		mockImageReader.On("UpdatePartial", mock.Anything, imageID, models.ImagePartial{
 			StudioID: models.NewOptionalInt(expectedStudioID),
 		}).Return(nil, nil).Once()
 	}
 
-	err := StudioImages(testCtx, &studio, nil, aliases, mockImageReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.StudioImages(testCtx, &studio, nil, aliases, mockImageReader)
 
 	assert := assert.New(t)
 
@@ -291,7 +308,9 @@ func testStudioGalleries(t *testing.T, tc testStudioCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedGalleryFilter := &models.GalleryFilterType{
 		Organized: &organized,
@@ -303,10 +322,12 @@ func testStudioGalleries(t *testing.T, tc testStudioCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
-	onNameQuery := mockGalleryReader.On("Query", testCtx, expectedGalleryFilter, expectedFindFilter)
+	onNameQuery := mockGalleryReader.On("Query", mock.Anything, expectedGalleryFilter, expectedFindFilter)
 	if aliasName == "" {
 		onNameQuery.Return(galleries, len(galleries), nil).Once()
 	} else {
@@ -320,18 +341,22 @@ func testStudioGalleries(t *testing.T, tc testStudioCase) {
 			},
 		}
 
-		mockGalleryReader.On("Query", testCtx, expectedAliasFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
+		mockGalleryReader.On("Query", mock.Anything, expectedAliasFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		galleryID := i + 1
 		expectedStudioID := studioID
-		mockGalleryReader.On("UpdatePartial", testCtx, galleryID, models.GalleryPartial{
+		mockGalleryReader.On("UpdatePartial", mock.Anything, galleryID, models.GalleryPartial{
 			StudioID: models.NewOptionalInt(expectedStudioID),
 		}).Return(nil, nil).Once()
 	}
 
-	err := StudioGalleries(testCtx, &studio, nil, aliases, mockGalleryReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.StudioGalleries(testCtx, &studio, nil, aliases, mockGalleryReader)
 
 	assert := assert.New(t)
 

internal/autotag/tag.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
 type SceneQueryTagUpdater interface {
@@ -50,8 +51,8 @@ func getTagTaggers(p *models.Tag, aliases []string, cache *match.Cache) []tagger
 }
 
 // TagScenes searches for scenes whose path matches the provided tag name and tags the scene with the tag.
-func TagScenes(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw SceneQueryTagUpdater, cache *match.Cache) error {
+func (tagger *Tagger) TagScenes(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw SceneQueryTagUpdater) error {
-	t := getTagTaggers(p, aliases, cache)
+	t := getTagTaggers(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagScenes(ctx, paths, rw, func(o *models.Scene) (bool, error) {
@@ -64,7 +65,9 @@ func TagScenes(ctx context.Context, p *models.Tag, paths []string, aliases []str
 				return false, nil
 			}
 
-			if err := scene.AddTag(ctx, rw, o, p.ID); err != nil {
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				return scene.AddTag(ctx, rw, o, p.ID)
+			}); err != nil {
 				return false, err
 			}
 
@@ -77,8 +80,8 @@ func TagScenes(ctx context.Context, p *models.Tag, paths []string, aliases []str
 }
 
 // TagImages searches for images whose path matches the provided tag name and tags the image with the tag.
-func TagImages(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw ImageQueryTagUpdater, cache *match.Cache) error {
+func (tagger *Tagger) TagImages(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw ImageQueryTagUpdater) error {
-	t := getTagTaggers(p, aliases, cache)
+	t := getTagTaggers(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagImages(ctx, paths, rw, func(o *models.Image) (bool, error) {
@@ -91,7 +94,9 @@ func TagImages(ctx context.Context, p *models.Tag, paths []string, aliases []str
 				return false, nil
 			}
 
-			if err := image.AddTag(ctx, rw, o, p.ID); err != nil {
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				return image.AddTag(ctx, rw, o, p.ID)
+			}); err != nil {
 				return false, err
 			}
 
@@ -104,8 +109,8 @@ func TagImages(ctx context.Context, p *models.Tag, paths []string, aliases []str
 }
 
 // TagGalleries searches for galleries whose path matches the provided tag name and tags the gallery with the tag.
-func TagGalleries(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw GalleryQueryTagUpdater, cache *match.Cache) error {
+func (tagger *Tagger) TagGalleries(ctx context.Context, p *models.Tag, paths []string, aliases []string, rw GalleryQueryTagUpdater) error {
-	t := getTagTaggers(p, aliases, cache)
+	t := getTagTaggers(p, aliases, tagger.Cache)
 
 	for _, tt := range t {
 		if err := tt.tagGalleries(ctx, paths, rw, func(o *models.Gallery) (bool, error) {
@@ -118,7 +123,9 @@ func TagGalleries(ctx context.Context, p *models.Tag, paths []string, aliases []
 				return false, nil
 			}
 
-			if err := gallery.AddTag(ctx, rw, o, p.ID); err != nil {
+			if err := txn.WithTxn(ctx, tagger.TxnManager, func(ctx context.Context) error {
+				return gallery.AddTag(ctx, rw, o, p.ID)
+			}); err != nil {
 				return false, err
 			}
 

internal/autotag/tag_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/stashapp/stash/pkg/models/mocks"
 	"github.com/stashapp/stash/pkg/scene"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 )
 
 type testTagCase struct {
@@ -111,7 +112,9 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedSceneFilter := &models.SceneFilterType{
 		Organized: &organized,
@@ -123,6 +126,8 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
@@ -140,13 +145,13 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 			},
 		}
 
-		mockSceneReader.On("Query", testCtx, scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+		mockSceneReader.On("Query", mock.Anything, scene.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
 			Return(mocks.SceneQueryResult(scenes, len(scenes)), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		sceneID := i + 1
-		mockSceneReader.On("UpdatePartial", testCtx, sceneID, models.ScenePartial{
+		mockSceneReader.On("UpdatePartial", mock.Anything, sceneID, models.ScenePartial{
 			TagIDs: &models.UpdateIDs{
 				IDs:  []int{tagID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -154,7 +159,11 @@ func testTagScenes(t *testing.T, tc testTagCase) {
 		}).Return(nil, nil).Once()
 	}
 
-	err := TagScenes(testCtx, &tag, nil, aliases, mockSceneReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.TagScenes(testCtx, &tag, nil, aliases, mockSceneReader)
 
 	assert := assert.New(t)
 
@@ -204,7 +213,9 @@ func testTagImages(t *testing.T, tc testTagCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedImageFilter := &models.ImageFilterType{
 		Organized: &organized,
@@ -216,6 +227,8 @@ func testTagImages(t *testing.T, tc testTagCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
@@ -233,14 +246,14 @@ func testTagImages(t *testing.T, tc testTagCase) {
 			},
 		}
 
-		mockImageReader.On("Query", testCtx, image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
+		mockImageReader.On("Query", mock.Anything, image.QueryOptions(expectedAliasFilter, expectedFindFilter, false)).
 			Return(mocks.ImageQueryResult(images, len(images)), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		imageID := i + 1
 
-		mockImageReader.On("UpdatePartial", testCtx, imageID, models.ImagePartial{
+		mockImageReader.On("UpdatePartial", mock.Anything, imageID, models.ImagePartial{
 			TagIDs: &models.UpdateIDs{
 				IDs:  []int{tagID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -248,7 +261,11 @@ func testTagImages(t *testing.T, tc testTagCase) {
 		}).Return(nil, nil).Once()
 	}
 
-	err := TagImages(testCtx, &tag, nil, aliases, mockImageReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.TagImages(testCtx, &tag, nil, aliases, mockImageReader)
 
 	assert := assert.New(t)
 
@@ -299,7 +316,9 @@ func testTagGalleries(t *testing.T, tc testTagCase) {
 	}
 
 	organized := false
-	perPage := models.PerPageAll
+	perPage := 1000
+	sort := "id"
+	direction := models.SortDirectionEnumAsc
 
 	expectedGalleryFilter := &models.GalleryFilterType{
 		Organized: &organized,
@@ -311,6 +330,8 @@ func testTagGalleries(t *testing.T, tc testTagCase) {
 
 	expectedFindFilter := &models.FindFilterType{
 		PerPage:   &perPage,
+		Sort:      &sort,
+		Direction: &direction,
 	}
 
 	// if alias provided, then don't find by name
@@ -328,13 +349,13 @@ func testTagGalleries(t *testing.T, tc testTagCase) {
 			},
 		}
 
-		mockGalleryReader.On("Query", testCtx, expectedAliasFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
+		mockGalleryReader.On("Query", mock.Anything, expectedAliasFilter, expectedFindFilter).Return(galleries, len(galleries), nil).Once()
 	}
 
 	for i := range matchingPaths {
 		galleryID := i + 1
 
-		mockGalleryReader.On("UpdatePartial", testCtx, galleryID, models.GalleryPartial{
+		mockGalleryReader.On("UpdatePartial", mock.Anything, galleryID, models.GalleryPartial{
 			TagIDs: &models.UpdateIDs{
 				IDs:  []int{tagID},
 				Mode: models.RelationshipUpdateModeAdd,
@@ -343,7 +364,11 @@ func testTagGalleries(t *testing.T, tc testTagCase) {
 
 	}
 
-	err := TagGalleries(testCtx, &tag, nil, aliases, mockGalleryReader, nil)
+	tagger := Tagger{
+		TxnManager: &mocks.TxnManager{},
+	}
+
+	err := tagger.TagGalleries(testCtx, &tag, nil, aliases, mockGalleryReader)
 
 	assert := assert.New(t)
 

internal/autotag/tagger.go

@@ -23,8 +23,14 @@ import (
 	"github.com/stashapp/stash/pkg/match"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
+	"github.com/stashapp/stash/pkg/txn"
 )
 
+type Tagger struct {
+	TxnManager txn.Manager
+	Cache      *match.Cache
+}
+
 type tagger struct {
 	ID      int
 	Type    string
@@ -112,12 +118,7 @@ func (t *tagger) tagTags(ctx context.Context, tagReader match.TagAutoTagQueryer,
 }
 
 func (t *tagger) tagScenes(ctx context.Context, paths []string, sceneReader scene.Queryer, addFunc addSceneLinkFunc) error {
-	others, err := match.PathToScenes(ctx, t.Name, paths, sceneReader)
+	return match.PathToScenesFn(ctx, t.Name, paths, sceneReader, func(ctx context.Context, p *models.Scene) error {
-	if err != nil {
-		return err
-	}
-
-	for _, p := range others {
 		added, err := addFunc(p)
 
 		if err != nil {
@@ -127,18 +128,13 @@ func (t *tagger) tagScenes(ctx context.Context, paths []string, sceneReader scen
 		if added {
 			t.addLog("scene", p.DisplayName())
 		}
-	}
 
 		return nil
+	})
 }
 
 func (t *tagger) tagImages(ctx context.Context, paths []string, imageReader image.Queryer, addFunc addImageLinkFunc) error {
-	others, err := match.PathToImages(ctx, t.Name, paths, imageReader)
+	return match.PathToImagesFn(ctx, t.Name, paths, imageReader, func(ctx context.Context, p *models.Image) error {
-	if err != nil {
-		return err
-	}
-
-	for _, p := range others {
 		added, err := addFunc(p)
 
 		if err != nil {
@@ -148,18 +144,13 @@ func (t *tagger) tagImages(ctx context.Context, paths []string, imageReader imag
 		if added {
 			t.addLog("image", p.DisplayName())
 		}
-	}
 
 		return nil
+	})
 }
 
 func (t *tagger) tagGalleries(ctx context.Context, paths []string, galleryReader gallery.Queryer, addFunc addGalleryLinkFunc) error {
-	others, err := match.PathToGalleries(ctx, t.Name, paths, galleryReader)
+	return match.PathToGalleriesFn(ctx, t.Name, paths, galleryReader, func(ctx context.Context, p *models.Gallery) error {
-	if err != nil {
-		return err
-	}
-
-	for _, p := range others {
 		added, err := addFunc(p)
 
 		if err != nil {
@@ -169,7 +160,7 @@ func (t *tagger) tagGalleries(ctx context.Context, paths []string, galleryReader
 		if added {
 			t.addLog("gallery", p.DisplayName())
 		}
-	}
 
 		return nil
+	})
 }

internal/manager/task_autotag.go

@@ -121,6 +121,11 @@ func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progre
 		return
 	}
 
+	tagger := autotag.Tagger{
+		TxnManager: j.txnManager,
+		Cache:      &j.cache,
+	}
+
 	for _, performerId := range performerIds {
 		var performers []*models.Performer
 
@@ -162,20 +167,20 @@ func (j *autoTagJob) autoTagPerformers(ctx context.Context, progress *job.Progre
 					return nil
 				}
 
-				if err := j.txnManager.WithTxn(ctx, func(ctx context.Context) error {
+				if err := func() error {
 					r := j.txnManager
-					if err := autotag.PerformerScenes(ctx, performer, paths, r.Scene, &j.cache); err != nil {
+					if err := tagger.PerformerScenes(ctx, performer, paths, r.Scene); err != nil {
 						return fmt.Errorf("processing scenes: %w", err)
 					}
-					if err := autotag.PerformerImages(ctx, performer, paths, r.Image, &j.cache); err != nil {
+					if err := tagger.PerformerImages(ctx, performer, paths, r.Image); err != nil {
 						return fmt.Errorf("processing images: %w", err)
 					}
-					if err := autotag.PerformerGalleries(ctx, performer, paths, r.Gallery, &j.cache); err != nil {
+					if err := tagger.PerformerGalleries(ctx, performer, paths, r.Gallery); err != nil {
 						return fmt.Errorf("processing galleries: %w", err)
 					}
 
 					return nil
-				}); err != nil {
+				}(); err != nil {
 					return fmt.Errorf("error auto-tagging performer '%s': %s", performer.Name, err.Error())
 				}
 
@@ -196,6 +201,10 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress,
 	}
 
 	r := j.txnManager
+	tagger := autotag.Tagger{
+		TxnManager: j.txnManager,
+		Cache:      &j.cache,
+	}
 
 	for _, studioId := range studioIds {
 		var studios []*models.Studio
@@ -238,24 +247,24 @@ func (j *autoTagJob) autoTagStudios(ctx context.Context, progress *job.Progress,
 					return nil
 				}
 
-				if err := j.txnManager.WithTxn(ctx, func(ctx context.Context) error {
+				if err := func() error {
 					aliases, err := r.Studio.GetAliases(ctx, studio.ID)
 					if err != nil {
 						return fmt.Errorf("getting studio aliases: %w", err)
 					}
 
-					if err := autotag.StudioScenes(ctx, studio, paths, aliases, r.Scene, &j.cache); err != nil {
+					if err := tagger.StudioScenes(ctx, studio, paths, aliases, r.Scene); err != nil {
 						return fmt.Errorf("processing scenes: %w", err)
 					}
-					if err := autotag.StudioImages(ctx, studio, paths, aliases, r.Image, &j.cache); err != nil {
+					if err := tagger.StudioImages(ctx, studio, paths, aliases, r.Image); err != nil {
 						return fmt.Errorf("processing images: %w", err)
 					}
-					if err := autotag.StudioGalleries(ctx, studio, paths, aliases, r.Gallery, &j.cache); err != nil {
+					if err := tagger.StudioGalleries(ctx, studio, paths, aliases, r.Gallery); err != nil {
 						return fmt.Errorf("processing galleries: %w", err)
 					}
 
 					return nil
-				}); err != nil {
+				}(); err != nil {
 					return fmt.Errorf("error auto-tagging studio '%s': %s", studio.Name.String, err.Error())
 				}
 
@@ -276,6 +285,10 @@ func (j *autoTagJob) autoTagTags(ctx context.Context, progress *job.Progress, pa
 	}
 
 	r := j.txnManager
+	tagger := autotag.Tagger{
+		TxnManager: j.txnManager,
+		Cache:      &j.cache,
+	}
 
 	for _, tagId := range tagIds {
 		var tags []*models.Tag
@@ -312,24 +325,24 @@ func (j *autoTagJob) autoTagTags(ctx context.Context, progress *job.Progress, pa
 					return nil
 				}
 
-				if err := j.txnManager.WithTxn(ctx, func(ctx context.Context) error {
+				if err := func() error {
 					aliases, err := r.Tag.GetAliases(ctx, tag.ID)
 					if err != nil {
 						return fmt.Errorf("getting tag aliases: %w", err)
 					}
 
-					if err := autotag.TagScenes(ctx, tag, paths, aliases, r.Scene, &j.cache); err != nil {
+					if err := tagger.TagScenes(ctx, tag, paths, aliases, r.Scene); err != nil {
 						return fmt.Errorf("processing scenes: %w", err)
 					}
-					if err := autotag.TagImages(ctx, tag, paths, aliases, r.Image, &j.cache); err != nil {
+					if err := tagger.TagImages(ctx, tag, paths, aliases, r.Image); err != nil {
 						return fmt.Errorf("processing images: %w", err)
 					}
-					if err := autotag.TagGalleries(ctx, tag, paths, aliases, r.Gallery, &j.cache); err != nil {
+					if err := tagger.TagGalleries(ctx, tag, paths, aliases, r.Gallery); err != nil {
 						return fmt.Errorf("processing galleries: %w", err)
 					}
 
 					return nil
-				}); err != nil {
+				}(); err != nil {
 					return fmt.Errorf("error auto-tagging tag '%s': %s", tag.Name, err.Error())
 				}
 

pkg/match/path.go

@@ -278,7 +278,7 @@ func PathToTags(ctx context.Context, path string, reader TagAutoTagQueryer, cach
 	return ret, nil
 }
 
-func PathToScenes(ctx context.Context, name string, paths []string, sceneReader scene.Queryer) ([]*models.Scene, error) {
+func PathToScenesFn(ctx context.Context, name string, paths []string, sceneReader scene.Queryer, fn func(ctx context.Context, scene *models.Scene) error) error {
 	regex := getPathQueryRegex(name)
 	organized := false
 	filter := models.SceneFilterType{
@@ -291,31 +291,53 @@ func PathToScenes(ctx context.Context, name string, paths []string, sceneReader
 
 	filter.And = scene.PathsFilter(paths)
 
-	pp := models.PerPageAll
+	// do in batches
+	pp := 1000
+	sort := "id"
+	sortDir := models.SortDirectionEnumAsc
+	lastID := 0
+
+	for {
+		if lastID != 0 {
+			filter.ID = &models.IntCriterionInput{
+				Value:    lastID,
+				Modifier: models.CriterionModifierGreaterThan,
+			}
+		}
+
 		scenes, err := scene.Query(ctx, sceneReader, &filter, &models.FindFilterType{
 			PerPage:   &pp,
+			Sort:      &sort,
+			Direction: &sortDir,
 		})
 
 		if err != nil {
-		return nil, fmt.Errorf("error querying scenes with regex '%s': %s", regex, err.Error())
+			return fmt.Errorf("error querying scenes with regex '%s': %s", regex, err.Error())
 		}
 
-	var ret []*models.Scene
-
 		// paths may have unicode characters
 		const useUnicode = true
 
 		r := nameToRegexp(name, useUnicode)
 		for _, p := range scenes {
 			if regexpMatchesPath(r, p.Path) != -1 {
-			ret = append(ret, p)
+				if err := fn(ctx, p); err != nil {
+					return fmt.Errorf("processing scene %s: %w", p.GetTitle(), err)
+				}
 			}
 		}
 
-	return ret, nil
+		if len(scenes) < pp {
+			break
 		}
 
-func PathToImages(ctx context.Context, name string, paths []string, imageReader image.Queryer) ([]*models.Image, error) {
+		lastID = scenes[len(scenes)-1].ID
+	}
+
+	return nil
+}
+
+func PathToImagesFn(ctx context.Context, name string, paths []string, imageReader image.Queryer, fn func(ctx context.Context, scene *models.Image) error) error {
 	regex := getPathQueryRegex(name)
 	organized := false
 	filter := models.ImageFilterType{
@@ -328,31 +350,53 @@ func PathToImages(ctx context.Context, name string, paths []string, imageReader
 
 	filter.And = image.PathsFilter(paths)
 
-	pp := models.PerPageAll
+	// do in batches
+	pp := 1000
+	sort := "id"
+	sortDir := models.SortDirectionEnumAsc
+	lastID := 0
+
+	for {
+		if lastID != 0 {
+			filter.ID = &models.IntCriterionInput{
+				Value:    lastID,
+				Modifier: models.CriterionModifierGreaterThan,
+			}
+		}
+
 		images, err := image.Query(ctx, imageReader, &filter, &models.FindFilterType{
 			PerPage:   &pp,
+			Sort:      &sort,
+			Direction: &sortDir,
 		})
 
 		if err != nil {
-		return nil, fmt.Errorf("error querying images with regex '%s': %s", regex, err.Error())
+			return fmt.Errorf("error querying images with regex '%s': %s", regex, err.Error())
 		}
 
-	var ret []*models.Image
-
 		// paths may have unicode characters
 		const useUnicode = true
 
 		r := nameToRegexp(name, useUnicode)
 		for _, p := range images {
 			if regexpMatchesPath(r, p.Path) != -1 {
-			ret = append(ret, p)
+				if err := fn(ctx, p); err != nil {
+					return fmt.Errorf("processing image %s: %w", p.GetTitle(), err)
+				}
 			}
 		}
 
-	return ret, nil
+		if len(images) < pp {
+			break
 		}
 
-func PathToGalleries(ctx context.Context, name string, paths []string, galleryReader gallery.Queryer) ([]*models.Gallery, error) {
+		lastID = images[len(images)-1].ID
+	}
+
+	return nil
+}
+
+func PathToGalleriesFn(ctx context.Context, name string, paths []string, galleryReader gallery.Queryer, fn func(ctx context.Context, scene *models.Gallery) error) error {
 	regex := getPathQueryRegex(name)
 	organized := false
 	filter := models.GalleryFilterType{
@@ -365,27 +409,49 @@ func PathToGalleries(ctx context.Context, name string, paths []string, galleryRe
 
 	filter.And = gallery.PathsFilter(paths)
 
-	pp := models.PerPageAll
+	// do in batches
-	gallerys, _, err := galleryReader.Query(ctx, &filter, &models.FindFilterType{
+	pp := 1000
+	sort := "id"
+	sortDir := models.SortDirectionEnumAsc
+	lastID := 0
+
+	for {
+		if lastID != 0 {
+			filter.ID = &models.IntCriterionInput{
+				Value:    lastID,
+				Modifier: models.CriterionModifierGreaterThan,
+			}
+		}
+
+		galleries, _, err := galleryReader.Query(ctx, &filter, &models.FindFilterType{
 			PerPage:   &pp,
+			Sort:      &sort,
+			Direction: &sortDir,
 		})
 
 		if err != nil {
-		return nil, fmt.Errorf("error querying gallerys with regex '%s': %s", regex, err.Error())
+			return fmt.Errorf("error querying galleries with regex '%s': %s", regex, err.Error())
 		}
 
-	var ret []*models.Gallery
-
 		// paths may have unicode characters
 		const useUnicode = true
 
 		r := nameToRegexp(name, useUnicode)
-	for _, p := range gallerys {
+		for _, p := range galleries {
 			path := p.Path
 			if path != "" && regexpMatchesPath(r, path) != -1 {
-			ret = append(ret, p)
+				if err := fn(ctx, p); err != nil {
+					return fmt.Errorf("processing gallery %s: %w", p.GetTitle(), err)
+				}
 			}
 		}
 
-	return ret, nil
+		if len(galleries) < pp {
+			break
+		}
+
+		lastID = galleries[len(galleries)-1].ID
+	}
+
+	return nil
 }

pkg/models/gallery.go

@@ -10,6 +10,7 @@ type GalleryFilterType struct {
 	And     *GalleryFilterType    `json:"AND"`
 	Or      *GalleryFilterType    `json:"OR"`
 	Not     *GalleryFilterType    `json:"NOT"`
+	ID      *IntCriterionInput    `json:"id"`
 	Title   *StringCriterionInput `json:"title"`
 	Details *StringCriterionInput `json:"details"`
 	// Filter by file checksum

pkg/models/image.go

@@ -6,6 +6,7 @@ type ImageFilterType struct {
 	And   *ImageFilterType      `json:"AND"`
 	Or    *ImageFilterType      `json:"OR"`
 	Not   *ImageFilterType      `json:"NOT"`
+	ID    *IntCriterionInput    `json:"id"`
 	Title *StringCriterionInput `json:"title"`
 	// Filter by file checksum
 	Checksum *StringCriterionInput `json:"checksum"`

pkg/models/scene.go

@@ -16,6 +16,7 @@ type SceneFilterType struct {
 	And      *SceneFilterType      `json:"AND"`
 	Or       *SceneFilterType      `json:"OR"`
 	Not      *SceneFilterType      `json:"NOT"`
+	ID       *IntCriterionInput    `json:"id"`
 	Title    *StringCriterionInput `json:"title"`
 	Code     *StringCriterionInput `json:"code"`
 	Details  *StringCriterionInput `json:"details"`

pkg/sqlite/gallery.go

@@ -624,6 +624,7 @@ func (qb *GalleryStore) makeFilter(ctx context.Context, galleryFilter *models.Ga
 		query.not(qb.makeFilter(ctx, galleryFilter.Not))
 	}
 
+	query.handleCriterion(ctx, intCriterionHandler(galleryFilter.ID, "galleries.id", nil))
 	query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.Title, "galleries.title"))
 	query.handleCriterion(ctx, stringCriterionHandler(galleryFilter.Details, "galleries.details"))
 

pkg/sqlite/image.go

@@ -619,6 +619,7 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 		query.not(qb.makeFilter(ctx, imageFilter.Not))
 	}
 
+	query.handleCriterion(ctx, intCriterionHandler(imageFilter.ID, "images.id", nil))
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if imageFilter.Checksum != nil {
 			qb.addImagesFilesTable(f)

pkg/sqlite/scene.go

@@ -806,6 +806,7 @@ func (qb *SceneStore) makeFilter(ctx context.Context, sceneFilter *models.SceneF
 		query.not(qb.makeFilter(ctx, sceneFilter.Not))
 	}
 
+	query.handleCriterion(ctx, intCriterionHandler(sceneFilter.ID, "scenes.id", nil))
 	query.handleCriterion(ctx, pathCriterionHandler(sceneFilter.Path, "folders.path", "files.basename", qb.addFoldersTable))
 	query.handleCriterion(ctx, sceneFileCountCriterionHandler(qb, sceneFilter.FileCount))
 	query.handleCriterion(ctx, stringCriterionHandler(sceneFilter.Title, "scenes.title"))

ui/v2.5/src/docs/en/Changelog/v0180.md

@@ -12,6 +12,7 @@
 * Changed Performer height to be numeric, and changed filtering accordingly. ([#3060](https://github.com/stashapp/stash/pull/3060))
 
 ### 🐛 Bug fixes
+* Fixed autotag error when tagging a large amount of objects. ([#3106](https://github.com/stashapp/stash/pull/3106))
 * Fixed Gallery title being incorrectly marked as mandatory for file- and folder-based galleries. ([#3110](https://github.com/stashapp/stash/pull/3110))
 * Fixed Saved Filters not ordered by name. ([#3101](https://github.com/stashapp/stash/pull/3101))
 * Scene Player no longer always resumes playing when seeking. ([#3020](https://github.com/stashapp/stash/pull/3020))