[Files Refactor] Performance tuning (#2865)

* Don't load image files by default
* Don't load gallery files by default
* Don't load scene files by default
* Retry locked transactions forever
* Don't show release notes if config not loaded
* Don't translate path slashes in export

pkg/scraper/autotag.go

@@ -95,7 +95,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 
 	// populate performers, studio and tags based on scene path
 	if err := txn.WithTxn(ctx, s.txnManager, func(ctx context.Context) error {
-		path := scene.Path()
+		path := scene.Path
 		performers, err := autotagMatchPerformers(ctx, path, s.performerReader, trimExt)
 		if err != nil {
 			return fmt.Errorf("autotag scraper viaScene: %w", err)
@@ -127,20 +127,20 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
 }
 
 func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
-	path := gallery.Path()
+	path := gallery.Path
 	if path == "" {
 		// not valid for non-path-based galleries
 		return nil, nil
 	}
 
 	// only trim extension if gallery is file-based
-	trimExt := gallery.PrimaryFile() != nil
+	trimExt := gallery.PrimaryFileID != nil
 
 	var ret *ScrapedGallery
 
 	// populate performers, studio and tags based on scene path
 	if err := txn.WithTxn(ctx, s.txnManager, func(ctx context.Context) error {
-		path := gallery.Path()
+		path := gallery.Path
 		performers, err := autotagMatchPerformers(ctx, path, s.performerReader, trimExt)
 		if err != nil {
 			return fmt.Errorf("autotag scraper viaGallery: %w", err)

pkg/scraper/cache.go

@@ -68,6 +68,7 @@ type TagFinder interface {
 
 type GalleryFinder interface {
 	Find(ctx context.Context, id int) (*models.Gallery, error)
+	models.FileLoader
 }
 
 type Repository struct {
@@ -364,6 +365,11 @@ func (c Cache) getGallery(ctx context.Context, galleryID int) (*models.Gallery,
 	if err := txn.WithTxn(ctx, c.txnManager, func(ctx context.Context) error {
 		var err error
 		ret, err = c.repository.GalleryFinder.Find(ctx, galleryID)
+
+		if ret != nil {
+			err = ret.LoadFiles(ctx, c.repository.GalleryFinder)
+		}
+
 		return err
 	}); err != nil {
 		return nil, err

pkg/scraper/query_url.go

@@ -13,9 +13,9 @@ type queryURLParameters map[string]string
 
 func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
 	ret := make(queryURLParameters)
-	ret["checksum"] = scene.Checksum()
-	ret["oshash"] = scene.OSHash()
-	ret["filename"] = filepath.Base(scene.Path())
+	ret["checksum"] = scene.Checksum
+	ret["oshash"] = scene.OSHash
+	ret["filename"] = filepath.Base(scene.Path)
 
 	if scene.Title != "" {
 		ret["title"] = scene.Title
@@ -53,8 +53,8 @@ func queryURLParametersFromGallery(gallery *models.Gallery) queryURLParameters {
 	ret := make(queryURLParameters)
 	ret["checksum"] = gallery.Checksum()
 
-	if gallery.Path() != "" {
-		ret["filename"] = filepath.Base(gallery.Path())
+	if gallery.Path != "" {
+		ret["filename"] = filepath.Base(gallery.Path)
 	}
 	if gallery.Title != "" {
 		ret["title"] = gallery.Title

pkg/scraper/stash.go

@@ -229,8 +229,8 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
 		Oshash   *string `graphql:"oshash" json:"oshash"`
 	}
 
-	checksum := scene.Checksum()
-	oshash := scene.OSHash()
+	checksum := scene.Checksum
+	oshash := scene.OSHash
 
 	input := SceneHashInput{
 		Checksum: &checksum,

pkg/scraper/stashbox/stash_box.go

@@ -17,6 +17,7 @@ import (
 	"golang.org/x/text/language"
 
 	"github.com/Yamashou/gqlgenc/graphqljson"
+	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/match"
@@ -33,6 +34,7 @@ import (
 type SceneReader interface {
 	Find(ctx context.Context, id int) (*models.Scene, error)
 	models.StashIDLoader
+	models.VideoFileLoader
 }
 
 type PerformerReader interface {
@@ -140,31 +142,37 @@ func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, ids []int)
 				return fmt.Errorf("scene with id %d not found", sceneID)
 			}
 
+			if err := scene.LoadFiles(ctx, c.repository.Scene); err != nil {
+				return err
+			}
+
 			var sceneFPs []*graphql.FingerprintQueryInput
 
-			checksum := scene.Checksum()
-			if checksum != "" {
-				sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-					Hash:      checksum,
-					Algorithm: graphql.FingerprintAlgorithmMd5,
-				})
-			}
+			for _, f := range scene.Files.List() {
+				checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5)
+				if checksum != "" {
+					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
+						Hash:      checksum,
+						Algorithm: graphql.FingerprintAlgorithmMd5,
+					})
+				}
 
-			oshash := scene.OSHash()
-			if oshash != "" {
-				sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-					Hash:      oshash,
-					Algorithm: graphql.FingerprintAlgorithmOshash,
-				})
-			}
+				oshash := f.Fingerprints.GetString(file.FingerprintTypeOshash)
+				if oshash != "" {
+					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
+						Hash:      oshash,
+						Algorithm: graphql.FingerprintAlgorithmOshash,
+					})
+				}
 
-			phash := scene.Phash()
-			if phash != 0 {
-				phashStr := utils.PhashToString(phash)
-				sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
-					Hash:      phashStr,
-					Algorithm: graphql.FingerprintAlgorithmPhash,
-				})
+				phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash)
+				if phash != 0 {
+					phashStr := utils.PhashToString(phash)
+					sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
+						Hash:      phashStr,
+						Algorithm: graphql.FingerprintAlgorithmPhash,
+					})
+				}
 			}
 
 			fingerprints = append(fingerprints, sceneFPs)
@@ -232,6 +240,10 @@ func (c Client) SubmitStashBoxFingerprints(ctx context.Context, sceneIDs []strin
 				return err
 			}
 
+			if err := scene.LoadFiles(ctx, qb); err != nil {
+				return err
+			}
+
 			stashIDs := scene.StashIDs.List()
 			sceneStashID := ""
 			for _, stashID := range stashIDs {
@@ -241,41 +253,46 @@ func (c Client) SubmitStashBoxFingerprints(ctx context.Context, sceneIDs []strin
 			}
 
 			if sceneStashID != "" {
-				duration := scene.Duration()
-				if checksum := scene.Checksum(); checksum != "" && duration != 0 {
-					fingerprint := graphql.FingerprintInput{
-						Hash:      checksum,
-						Algorithm: graphql.FingerprintAlgorithmMd5,
-						Duration:  int(duration),
-					}
-					fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-						SceneID:     sceneStashID,
-						Fingerprint: &fingerprint,
-					})
-				}
+				for _, f := range scene.Files.List() {
+					duration := f.Duration
 
-				if oshash := scene.OSHash(); oshash != "" && duration != 0 {
-					fingerprint := graphql.FingerprintInput{
-						Hash:      oshash,
-						Algorithm: graphql.FingerprintAlgorithmOshash,
-						Duration:  int(duration),
-					}
-					fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-						SceneID:     sceneStashID,
-						Fingerprint: &fingerprint,
-					})
-				}
+					if duration != 0 {
+						if checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5); checksum != "" {
+							fingerprint := graphql.FingerprintInput{
+								Hash:      checksum,
+								Algorithm: graphql.FingerprintAlgorithmMd5,
+								Duration:  int(duration),
+							}
+							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
+								SceneID:     sceneStashID,
+								Fingerprint: &fingerprint,
+							})
+						}
 
-				if phash := scene.Phash(); phash != 0 && duration != 0 {
-					fingerprint := graphql.FingerprintInput{
-						Hash:      utils.PhashToString(phash),
-						Algorithm: graphql.FingerprintAlgorithmPhash,
-						Duration:  int(duration),
+						if oshash := f.Fingerprints.GetString(file.FingerprintTypeOshash); oshash != "" {
+							fingerprint := graphql.FingerprintInput{
+								Hash:      oshash,
+								Algorithm: graphql.FingerprintAlgorithmOshash,
+								Duration:  int(duration),
+							}
+							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
+								SceneID:     sceneStashID,
+								Fingerprint: &fingerprint,
+							})
+						}
+
+						if phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash); phash != 0 {
+							fingerprint := graphql.FingerprintInput{
+								Hash:      utils.PhashToString(phash),
+								Algorithm: graphql.FingerprintAlgorithmPhash,
+								Duration:  int(duration),
+							}
+							fingerprints = append(fingerprints, graphql.FingerprintSubmission{
+								SceneID:     sceneStashID,
+								Fingerprint: &fingerprint,
+							})
+						}
 					}
-					fingerprints = append(fingerprints, graphql.FingerprintSubmission{
-						SceneID:     sceneStashID,
-						Fingerprint: &fingerprint,
-					})
 				}
 			}
 		}
@@ -778,7 +795,7 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 		}
 		for _, stashID := range stashIDs {
 			c := stashID
-			if c.Endpoint == endpoint {
+			if stashID.Endpoint == endpoint {
 				studioDraft.ID = &c.StashID
 				break
 			}
@@ -787,32 +804,39 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 	}
 
 	fingerprints := []*graphql.FingerprintInput{}
-	duration := scene.Duration()
-	if oshash := scene.OSHash(); oshash != "" && duration != 0 {
-		fingerprint := graphql.FingerprintInput{
-			Hash:      oshash,
-			Algorithm: graphql.FingerprintAlgorithmOshash,
-			Duration:  int(duration),
-		}
-		fingerprints = append(fingerprints, &fingerprint)
-	}
 
-	if checksum := scene.Checksum(); checksum != "" && duration != 0 {
-		fingerprint := graphql.FingerprintInput{
-			Hash:      checksum,
-			Algorithm: graphql.FingerprintAlgorithmMd5,
-			Duration:  int(duration),
-		}
-		fingerprints = append(fingerprints, &fingerprint)
-	}
+	// submit all file fingerprints
+	for _, f := range scene.Files.List() {
+		duration := f.Duration
 
-	if phash := scene.Phash(); phash != 0 && duration != 0 {
-		fingerprint := graphql.FingerprintInput{
-			Hash:      utils.PhashToString(phash),
-			Algorithm: graphql.FingerprintAlgorithmPhash,
-			Duration:  int(duration),
+		if duration != 0 {
+			if oshash := f.Fingerprints.GetString(file.FingerprintTypeOshash); oshash != "" {
+				fingerprint := graphql.FingerprintInput{
+					Hash:      oshash,
+					Algorithm: graphql.FingerprintAlgorithmOshash,
+					Duration:  int(duration),
+				}
+				fingerprints = append(fingerprints, &fingerprint)
+			}
+
+			if checksum := f.Fingerprints.GetString(file.FingerprintTypeMD5); checksum != "" {
+				fingerprint := graphql.FingerprintInput{
+					Hash:      checksum,
+					Algorithm: graphql.FingerprintAlgorithmMd5,
+					Duration:  int(duration),
+				}
+				fingerprints = append(fingerprints, &fingerprint)
+			}
+
+			if phash := f.Fingerprints.GetInt64(file.FingerprintTypePhash); phash != 0 {
+				fingerprint := graphql.FingerprintInput{
+					Hash:      utils.PhashToString(phash),
+					Algorithm: graphql.FingerprintAlgorithmPhash,
+					Duration:  int(duration),
+				}
+				fingerprints = append(fingerprints, &fingerprint)
+			}
 		}
-		fingerprints = append(fingerprints, &fingerprint)
 	}
 	draft.Fingerprints = fingerprints
 
@@ -854,11 +878,13 @@ func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, endpo
 	}
 	draft.Tags = tags
 
-	exists, _ := fsutil.FileExists(imagePath)
-	if exists {
-		file, err := os.Open(imagePath)
-		if err == nil {
-			image = file
+	if imagePath != "" {
+		exists, _ := fsutil.FileExists(imagePath)
+		if exists {
+			file, err := os.Open(imagePath)
+			if err == nil {
+				image = file
+			}
 		}
 	}