[Files Refactor] Performance tuning (#2784)

* Improve image query performance
* Tune queries
* Fix db generator
* Don't show release notes in setup
* Further tune indexes
* Log when creating screenshot

scripts/test_db_generator/makeTestDB.go

@@ -1,5 +1,5 @@
-//go:build ignore
-// +build ignore
+//go:build tools
+// +build tools
 
 package main
 
@@ -14,12 +14,12 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/stashapp/stash/pkg/database"
+	"github.com/stashapp/stash/pkg/file"
 	"github.com/stashapp/stash/pkg/hash/md5"
-	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/sliceutil/intslice"
 	"github.com/stashapp/stash/pkg/sqlite"
+	"github.com/stashapp/stash/pkg/txn"
 	"gopkg.in/yaml.v2"
 )
 
@@ -40,8 +40,12 @@ type config struct {
 	Naming     namingConfig `yaml:"naming"`
 }
 
-var txnManager models.TransactionManager
-var c *config
+var (
+	repo     models.Repository
+	c        *config
+	db       *sqlite.Database
+	folderID file.FolderID
+)
 
 func main() {
 	rand.Seed(time.Now().UnixNano())
@@ -54,9 +58,17 @@ func main() {
 
 	initNaming(*c)
 
-	if err = database.Initialize(c.Database); err != nil {
+	db = sqlite.NewDatabase()
+	repo = db.TxnRepository()
+
+	logf("Initializing database...")
+	if err = db.Open(c.Database); err != nil {
 		log.Fatalf("couldn't initialize database: %v", err)
 	}
+	logf("Populating database...")
+	if err = makeFolder(); err != nil {
+		log.Fatalf("couldn't create folder: %v", err)
+	}
 	populateDB()
 }
 
@@ -89,12 +101,8 @@ func populateDB() {
 	makeMarkers(c.Markers)
 }
 
-func withTxn(f func(r models.Repository) error) error {
-	if txnManager == nil {
-		txnManager = sqlite.NewTransactionManager()
-	}
-
-	return txnManager.WithTxn(context.TODO(), f)
+func withTxn(f func(ctx context.Context) error) error {
+	return txn.WithTxn(context.Background(), db, f)
 }
 
 func retry(attempts int, fn func() error) error {
@@ -109,28 +117,43 @@ func retry(attempts int, fn func() error) error {
 	return err
 }
 
+func makeFolder() error {
+	return withTxn(func(ctx context.Context) error {
+		f := file.Folder{
+			Path: ".",
+		}
+		if err := repo.Folder.Create(ctx, &f); err != nil {
+			return err
+		}
+
+		folderID = f.ID
+		return nil
+	})
+}
+
 func makeTags(n int) {
+	logf("creating %d tags...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := names[c.Naming.Tags].generateName(1)
 				tag := models.Tag{
 					Name: name,
 				}
 
-				created, err := r.Tag().Create(tag)
+				created, err := repo.Tag.Create(ctx, tag)
 				if err != nil {
 					return err
 				}
 
 				if rand.Intn(100) > 5 {
-					t, _, err := r.Tag().Query(nil, getRandomFilter(1))
+					t, _, err := repo.Tag.Query(ctx, nil, getRandomFilter(1))
 					if err != nil {
 						return err
 					}
 
 					if len(t) > 0 && t[0].ID != created.ID {
-						if err := r.Tag().UpdateParentTags(created.ID, []int{t[0].ID}); err != nil {
+						if err := repo.Tag.UpdateParentTags(ctx, created.ID, []int{t[0].ID}); err != nil {
 							return err
 						}
 					}
@@ -145,9 +168,10 @@ func makeTags(n int) {
 }
 
 func makeStudios(n int) {
+	logf("creating %d studios...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := names[c.Naming.Tags].generateName(rand.Intn(5) + 1)
 				studio := models.Studio{
 					Name:     sql.NullString{String: name, Valid: true},
@@ -155,7 +179,7 @@ func makeStudios(n int) {
 				}
 
 				if rand.Intn(100) > 5 {
-					ss, _, err := r.Studio().Query(nil, getRandomFilter(1))
+					ss, _, err := repo.Studio.Query(ctx, nil, getRandomFilter(1))
 					if err != nil {
 						return err
 					}
@@ -168,7 +192,7 @@ func makeStudios(n int) {
 					}
 				}
 
-				_, err := r.Studio().Create(studio)
+				_, err := repo.Studio.Create(ctx, studio)
 				return err
 			})
 		}); err != nil {
@@ -178,9 +202,10 @@ func makeStudios(n int) {
 }
 
 func makePerformers(n int) {
+	logf("creating %d performers...", n)
 	for i := 0; i < n; i++ {
 		if err := retry(100, func() error {
-			return withTxn(func(r models.Repository) error {
+			return withTxn(func(ctx context.Context) error {
 				name := generatePerformerName()
 				performer := models.Performer{
 					Name:     sql.NullString{String: name, Valid: true},
@@ -193,7 +218,7 @@ func makePerformers(n int) {
 
 				// TODO - set tags
 
-				_, err := r.Performer().Create(performer)
+				_, err := repo.Performer.Create(ctx, performer)
 				if err != nil {
 					err = fmt.Errorf("error creating performer with name: %s: %s", performer.Name.String, err.Error())
 				}
@@ -205,23 +230,72 @@ func makePerformers(n int) {
 	}
 }
 
+func generateBaseFile(path string) *file.BaseFile {
+	return &file.BaseFile{
+		Path:           path,
+		Basename:       path,
+		ParentFolderID: folderID,
+		Fingerprints: []file.Fingerprint{
+			file.Fingerprint{
+				Type:        "md5",
+				Fingerprint: md5.FromString(path),
+			},
+			file.Fingerprint{
+				Type:        "oshash",
+				Fingerprint: md5.FromString(path),
+			},
+		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
+	}
+}
+
+func generateVideoFile(path string) file.File {
+	w, h := getResolution()
+
+	return &file.VideoFile{
+		BaseFile: generateBaseFile(path),
+		Duration: rand.Float64() * 14400,
+		Height:   h,
+		Width:    w,
+	}
+}
+
+func makeVideoFile(ctx context.Context, path string) (file.File, error) {
+	f := generateVideoFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func logf(f string, args ...interface{}) {
+	log.Printf(f+"\n", args...)
+}
+
 func makeScenes(n int) {
-	logger.Infof("creating %d scenes...", n)
+	logf("creating %d scenes...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
 
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				scene := generateScene(i)
-				scene.StudioID = getRandomStudioID(r)
+				scene.StudioID = getRandomStudioID(ctx)
+				makeSceneRelationships(ctx, &scene)
 
-				created, err := r.Scene().Create(scene)
+				path := md5.FromString("scene/" + strconv.Itoa(i))
+				f, err := makeVideoFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeSceneRelationships(r, created.ID)
+				if err := repo.Scene.Create(ctx, &scene, []file.ID{f.Base().ID}); err != nil {
+					return err
+				}
 			}
 
 			return nil
@@ -229,14 +303,14 @@ func makeScenes(n int) {
 			panic(err)
 		}
 
-		logger.Infof("... created %d scenes", i)
+		logf("... created %d scenes", i)
 	}
 }
 
-func getResolution() (int64, int64) {
+func getResolution() (int, int) {
 	res := models.AllResolutionEnum[rand.Intn(len(models.AllResolutionEnum))]
-	h := int64(res.GetMaxResolution())
-	var w int64
+	h := res.GetMaxResolution()
+	var w int
 	if h == 240 || h == 480 || rand.Intn(10) == 9 {
 		w = h * 4 / 3
 	} else {
@@ -250,54 +324,69 @@ func getResolution() (int64, int64) {
 	return w, h
 }
 
-func getDate() string {
+func getDate() time.Time {
 	s := rand.Int63n(time.Now().Unix())
 
-	d := time.Unix(s, 0)
-	return d.Format("2006-01-02")
+	return time.Unix(s, 0)
 }
 
 func generateScene(i int) models.Scene {
-	path := md5.FromString("scene/" + strconv.Itoa(i))
-	w, h := getResolution()
-
 	return models.Scene{
-		Path:     path,
-		Title:    sql.NullString{String: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1), Valid: true},
-		Checksum: sql.NullString{String: md5.FromString(path), Valid: true},
-		OSHash:   sql.NullString{String: md5.FromString(path), Valid: true},
-		Duration: sql.NullFloat64{
-			Float64: rand.Float64() * 14400,
-			Valid:   true,
-		},
-		Height: models.NullInt64(h),
-		Width:  models.NullInt64(w),
-		Date: models.SQLiteDate{
-			String: getDate(),
-			Valid:  true,
+		Title: names[c.Naming.Scenes].generateName(rand.Intn(7) + 1),
+		Date: &models.Date{
+			Time: getDate(),
 		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 	}
 }
 
+func generateImageFile(path string) file.File {
+	w, h := getResolution()
+
+	return &file.ImageFile{
+		BaseFile: generateBaseFile(path),
+		Height:   h,
+		Width:    w,
+	}
+}
+
+func makeImageFile(ctx context.Context, path string) (file.File, error) {
+	f := generateImageFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
 func makeImages(n int) {
-	logger.Infof("creating %d images...", n)
+	logf("creating %d images...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				image := generateImage(i)
-				image.StudioID = getRandomStudioID(r)
+				image.StudioID = getRandomStudioID(ctx)
+				makeImageRelationships(ctx, &image)
 
-				created, err := r.Image().Create(image)
+				path := md5.FromString("image/" + strconv.Itoa(i))
+				f, err := makeImageFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeImageRelationships(r, created.ID)
+				if err := repo.Image.Create(ctx, &models.ImageCreateInput{
+					Image:   &image,
+					FileIDs: []file.ID{f.Base().ID},
+				}); err != nil {
+					return err
+				}
 			}
 
-			logger.Infof("... created %d images", i)
+			logf("... created %d images", i)
 
 			return nil
 		}); err != nil {
@@ -307,36 +396,37 @@ func makeImages(n int) {
 }
 
 func generateImage(i int) models.Image {
-	path := md5.FromString("image/" + strconv.Itoa(i))
-
-	w, h := getResolution()
-
 	return models.Image{
-		Title:    sql.NullString{String: names[c.Naming.Images].generateName(rand.Intn(7) + 1), Valid: true},
-		Path:     path,
-		Checksum: md5.FromString(path),
-		Height:   models.NullInt64(h),
-		Width:    models.NullInt64(w),
+		Title:     names[c.Naming.Images].generateName(rand.Intn(7) + 1),
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 	}
 }
 
 func makeGalleries(n int) {
-	logger.Infof("creating %d galleries...", n)
+	logf("creating %d galleries...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
 
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				gallery := generateGallery(i)
-				gallery.StudioID = getRandomStudioID(r)
+				gallery.StudioID = getRandomStudioID(ctx)
+				gallery.TagIDs = getRandomTags(ctx, 0, 15)
+				gallery.PerformerIDs = getRandomPerformers(ctx)
 
-				created, err := r.Gallery().Create(gallery)
+				path := md5.FromString("gallery/" + strconv.Itoa(i))
+				f, err := makeZipFile(ctx, path)
 				if err != nil {
 					return err
 				}
 
-				makeGalleryRelationships(r, created.ID)
+				if err := repo.Gallery.Create(ctx, &gallery, []file.ID{f.Base().ID}); err != nil {
+					return err
+				}
+
+				makeGalleryRelationships(ctx, &gallery)
 			}
 
 			return nil
@@ -344,49 +434,60 @@ func makeGalleries(n int) {
 			panic(err)
 		}
 
-		logger.Infof("... created %d galleries", i)
+		logf("... created %d galleries", i)
 	}
 }
 
-func generateGallery(i int) models.Gallery {
-	path := md5.FromString("gallery/" + strconv.Itoa(i))
+func generateZipFile(path string) file.File {
+	return generateBaseFile(path)
+}
 
+func makeZipFile(ctx context.Context, path string) (file.File, error) {
+	f := generateZipFile(path)
+
+	if err := repo.File.Create(ctx, f); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func generateGallery(i int) models.Gallery {
 	return models.Gallery{
-		Title:    sql.NullString{String: names[c.Naming.Galleries].generateName(rand.Intn(7) + 1), Valid: true},
-		Path:     sql.NullString{String: path, Valid: true},
-		Checksum: md5.FromString(path),
-		Date: models.SQLiteDate{
-			String: getDate(),
-			Valid:  true,
+		Title: names[c.Naming.Galleries].generateName(rand.Intn(7) + 1),
+		Date: &models.Date{
+			Time: getDate(),
 		},
+		CreatedAt: time.Now(),
+		UpdatedAt: time.Now(),
 	}
 }
 
 func makeMarkers(n int) {
-	logger.Infof("creating %d markers...", n)
+	logf("creating %d markers...", n)
 	for i := 0; i < n; {
 		// do in batches of 1000
 		batch := i + batchSize
-		if err := withTxn(func(r models.Repository) error {
+		if err := withTxn(func(ctx context.Context) error {
 			for ; i < batch && i < n; i++ {
 				marker := generateMarker(i)
 				marker.SceneID = models.NullInt64(int64(getRandomScene()))
-				marker.PrimaryTagID = getRandomTags(r, 1, 1)[0]
+				marker.PrimaryTagID = getRandomTags(ctx, 1, 1)[0]
 
-				created, err := r.SceneMarker().Create(marker)
+				created, err := repo.SceneMarker.Create(ctx, marker)
 				if err != nil {
 					return err
 				}
 
-				tags := getRandomTags(r, 0, 5)
+				tags := getRandomTags(ctx, 0, 5)
 				// remove primary tag
 				tags = intslice.IntExclude(tags, []int{marker.PrimaryTagID})
-				if err := r.SceneMarker().UpdateTags(created.ID, tags); err != nil {
+				if err := repo.SceneMarker.UpdateTags(ctx, created.ID, tags); err != nil {
 					return err
 				}
 			}
 
-			logger.Infof("... created %d markers", i)
+			logf("... created %d markers", i)
 
 			return nil
 		}); err != nil {
@@ -410,9 +511,9 @@ func getRandomFilter(n int) *models.FindFilterType {
 	}
 }
 
-func getRandomStudioID(r models.Repository) sql.NullInt64 {
+func getRandomStudioID(ctx context.Context) *int {
 	if rand.Intn(10) == 0 {
-		return sql.NullInt64{}
+		return nil
 	}
 
 	// s, _, err := r.Studio().Query(nil, getRandomFilter(1))
@@ -420,82 +521,44 @@ func getRandomStudioID(r models.Repository) sql.NullInt64 {
 	// 	panic(err)
 	// }
 
-	return sql.NullInt64{
-		Int64: int64(rand.Int63n(int64(c.Studios)) + 1),
-		Valid: true,
-	}
+	v := rand.Intn(c.Studios) + 1
+	return &v
 }
 
-func makeSceneRelationships(r models.Repository, id int) {
+func makeSceneRelationships(ctx context.Context, s *models.Scene) {
 	// add tags
-	tagIDs := getRandomTags(r, 0, 15)
-	if len(tagIDs) > 0 {
-		if err := r.Scene().UpdateTags(id, tagIDs); err != nil {
-			panic(err)
-		}
-	}
+	s.TagIDs = getRandomTags(ctx, 0, 15)
 
 	// add performers
-	performerIDs := getRandomPerformers(r)
-	if len(tagIDs) > 0 {
-		if err := r.Scene().UpdatePerformers(id, performerIDs); err != nil {
-			panic(err)
-		}
-	}
+	s.PerformerIDs = getRandomPerformers(ctx)
 }
 
-func makeImageRelationships(r models.Repository, id int) {
+func makeImageRelationships(ctx context.Context, i *models.Image) {
 	// there are typically many more images. For performance reasons
 	// only a small proportion should have tags/performers
 
 	// add tags
 	if rand.Intn(100) == 0 {
-		tagIDs := getRandomTags(r, 1, 15)
-		if len(tagIDs) > 0 {
-			if err := r.Image().UpdateTags(id, tagIDs); err != nil {
-				panic(err)
-			}
-		}
+		i.TagIDs = getRandomTags(ctx, 1, 15)
 	}
 
 	// add performers
 	if rand.Intn(100) <= 1 {
-		performerIDs := getRandomPerformers(r)
-		if len(performerIDs) > 0 {
-			if err := r.Image().UpdatePerformers(id, performerIDs); err != nil {
-				panic(err)
-			}
-		}
+		i.PerformerIDs = getRandomPerformers(ctx)
 	}
 }
 
-func makeGalleryRelationships(r models.Repository, id int) {
-	// add tags
-	tagIDs := getRandomTags(r, 0, 15)
-	if len(tagIDs) > 0 {
-		if err := r.Gallery().UpdateTags(id, tagIDs); err != nil {
-			panic(err)
-		}
-	}
-
-	// add performers
-	performerIDs := getRandomPerformers(r)
-	if len(tagIDs) > 0 {
-		if err := r.Gallery().UpdatePerformers(id, performerIDs); err != nil {
-			panic(err)
-		}
-	}
-
+func makeGalleryRelationships(ctx context.Context, g *models.Gallery) {
 	// add images
-	imageIDs := getRandomImages(r)
-	if len(tagIDs) > 0 {
-		if err := r.Gallery().UpdateImages(id, imageIDs); err != nil {
+	imageIDs := getRandomImages(ctx)
+	if len(imageIDs) > 0 {
+		if err := repo.Gallery.UpdateImages(ctx, g.ID, imageIDs); err != nil {
 			panic(err)
 		}
 	}
 }
 
-func getRandomPerformers(r models.Repository) []int {
+func getRandomPerformers(ctx context.Context) []int {
 	n := rand.Intn(5)
 
 	var ret []int
@@ -521,7 +584,7 @@ func getRandomScene() int {
 	return rand.Intn(c.Scenes) + 1
 }
 
-func getRandomTags(r models.Repository, min, max int) []int {
+func getRandomTags(ctx context.Context, min, max int) []int {
 	var n int
 	if min == max {
 		n = min
@@ -548,7 +611,7 @@ func getRandomTags(r models.Repository, min, max int) []int {
 	return ret
 }
 
-func getRandomImages(r models.Repository) []int {
+func getRandomImages(ctx context.Context) []int {
 	n := rand.Intn(500)
 
 	var ret []int

scripts/test_db_generator/naming.go

@@ -1,4 +1,5 @@
-// +build ignore
+//go:build tools
+// +build tools
 
 package main