File storage rewrite (#2676)

* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes

internal/manager/task_generate.go

@@ -2,7 +2,6 @@ package manager
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"time"
 
@@ -54,7 +53,7 @@ type GeneratePreviewOptionsInput struct {
 const generateQueueSize = 200000
 
 type GenerateJob struct {
-	txnManager models.Repository
+	txnManager Repository
 	input      GenerateMetadataInput
 
 	overwrite      bool
@@ -192,36 +191,29 @@ func (j *GenerateJob) queueTasks(ctx context.Context, g *generate.Generator, que
 
 	findFilter := models.BatchFindFilter(batchSize)
 
-	if err := j.txnManager.WithTxn(ctx, func(ctx context.Context) error {
-		for more := true; more; {
-			if job.IsCancelled(ctx) {
-				return context.Canceled
-			}
-
-			scenes, err := scene.Query(ctx, j.txnManager.Scene, nil, findFilter)
-			if err != nil {
-				return err
-			}
-
-			for _, ss := range scenes {
-				if job.IsCancelled(ctx) {
-					return context.Canceled
-				}
-
-				j.queueSceneJobs(ctx, g, ss, queue, &totals)
-			}
-
-			if len(scenes) != batchSize {
-				more = false
-			} else {
-				*findFilter.Page++
-			}
+	for more := true; more; {
+		if job.IsCancelled(ctx) {
+			return totals
 		}
 
-		return nil
-	}); err != nil {
-		if !errors.Is(err, context.Canceled) {
+		scenes, err := scene.Query(ctx, j.txnManager.Scene, nil, findFilter)
+		if err != nil {
 			logger.Errorf("Error encountered queuing files to scan: %s", err.Error())
+			return totals
+		}
+
+		for _, ss := range scenes {
+			if job.IsCancelled(ctx) {
+				return totals
+			}
+
+			j.queueSceneJobs(ctx, g, ss, queue, &totals)
+		}
+
+		if len(scenes) != batchSize {
+			more = false
+		} else {
+			*findFilter.Page++
 		}
 	}
 
@@ -351,17 +343,21 @@ func (j *GenerateJob) queueSceneJobs(ctx context.Context, g *generate.Generator,
 	}
 
 	if utils.IsTrue(j.input.Phashes) {
-		task := &GeneratePhashTask{
-			Scene:               *scene,
-			fileNamingAlgorithm: j.fileNamingAlgo,
-			txnManager:          j.txnManager,
-			Overwrite:           j.overwrite,
-		}
+		// generate for all files in scene
+		for _, f := range scene.Files {
+			task := &GeneratePhashTask{
+				File:                f,
+				fileNamingAlgorithm: j.fileNamingAlgo,
+				txnManager:          j.txnManager,
+				fileUpdater:         j.txnManager.File,
+				Overwrite:           j.overwrite,
+			}
 
-		if task.shouldGenerate() {
-			totals.phashes++
-			totals.tasks++
-			queue <- task
+			if task.shouldGenerate() {
+				totals.phashes++
+				totals.tasks++
+				queue <- task
+			}
 		}
 	}