[Files Refactor] Performance tuning (#2784)

* Improve image query performance
* Tune queries
* Fix db generator
* Don't show release notes in setup
* Further tune indexes
* Log when creating screenshot

pkg/sqlite/image.go

@@ -3,9 +3,7 @@ package sqlite
 import (
 	"context"
 	"database/sql"
-	"errors"
 	"fmt"
-	"path/filepath"
 	"time"
 
 	"github.com/jmoiron/sqlx"
@@ -365,11 +363,14 @@ func (qb *ImageStore) findBySubquery(ctx context.Context, sq *goqu.SelectDataset
 }
 
 func (qb *ImageStore) FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error) {
-	table := imagesQueryTable
+	table := qb.table()
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("file_id").Eq(fileID),
-	)
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileID))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -387,18 +388,28 @@ func (qb *ImageStore) CountByFileID(ctx context.Context, fileID file.ID) (int, e
 }
 
 func (qb *ImageStore) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error) {
-	table := imagesQueryTable
+	table := qb.table()
+	fingerprintTable := fingerprintTableMgr.table
 
 	var ex []exp.Expression
 
 	for _, v := range fp {
 		ex = append(ex, goqu.And(
-			table.Col("fingerprint_type").Eq(v.Type),
-			table.Col("fingerprint").Eq(v.Fingerprint),
+			fingerprintTable.Col("type").Eq(v.Type),
+			fingerprintTable.Col("fingerprint").Eq(v.Fingerprint),
 		))
 	}
 
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(goqu.Or(ex...))
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fingerprintTable,
+			goqu.On(fingerprintTable.Col(fileIDColumn).Eq(imagesFilesJoinTable.Col(fileIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(goqu.Or(ex...))
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -409,45 +420,32 @@ func (qb *ImageStore) FindByFingerprints(ctx context.Context, fp []file.Fingerpr
 }
 
 func (qb *ImageStore) FindByChecksum(ctx context.Context, checksum string) ([]*models.Image, error) {
-	table := imagesQueryTable
-
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("fingerprint_type").Eq(file.FingerprintTypeMD5),
-		table.Col("fingerprint").Eq(checksum),
-	)
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil {
-		return nil, fmt.Errorf("getting image by checksum %s: %w", checksum, err)
-	}
-
-	return ret, nil
-}
-
-func (qb *ImageStore) FindByPath(ctx context.Context, p string) ([]*models.Image, error) {
-	table := imagesQueryTable
-	basename := filepath.Base(p)
-	dir, _ := path(filepath.Dir(p)).Value()
-
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(
-		table.Col("parent_folder_path").Eq(dir),
-		table.Col("basename").Eq(basename),
-	)
-
-	ret, err := qb.findBySubquery(ctx, sq)
-	if err != nil && !errors.Is(err, sql.ErrNoRows) {
-		return nil, fmt.Errorf("getting image by path %s: %w", p, err)
-	}
-
-	return ret, nil
+	return qb.FindByFingerprints(ctx, []file.Fingerprint{
+		{
+			Type:        file.FingerprintTypeMD5,
+			Fingerprint: checksum,
+		},
+	})
 }
 
 func (qb *ImageStore) FindByGalleryID(ctx context.Context, galleryID int) ([]*models.Image, error) {
-	table := qb.queryTable()
+	table := qb.table()
+	queryTable := qb.queryTable()
 
-	q := qb.selectDataset().Where(
-		table.Col("gallery_id").Eq(galleryID),
-	).GroupBy(table.Col(idColumn)).Order(table.Col("parent_folder_path").Asc(), table.Col("basename").Asc())
+	sq := dialect.From(table).
+		InnerJoin(
+			galleriesImagesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(galleriesImagesJoinTable.Col(imageIDColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		galleriesImagesJoinTable.Col("gallery_id").Eq(galleryID),
+	)
+
+	q := qb.selectDataset().Prepared(true).Where(
+		queryTable.Col(idColumn).Eq(
+			sq,
+		),
+	).Order(queryTable.Col("parent_folder_path").Asc(), queryTable.Col("basename").Asc())
 
 	ret, err := qb.getMany(ctx, q)
 	if err != nil {
@@ -465,8 +463,21 @@ func (qb *ImageStore) CountByGalleryID(ctx context.Context, galleryID int) (int,
 }
 
 func (qb *ImageStore) FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Image, error) {
-	table := qb.queryTable()
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(table.Col("parent_folder_id").Eq(folderID))
+	table := qb.table()
+	fileTable := goqu.T(fileTable)
+
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fileTable,
+			goqu.On(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileTable.Col(idColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		fileTable.Col("parent_folder_id").Eq(folderID),
+	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -477,8 +488,21 @@ func (qb *ImageStore) FindByFolderID(ctx context.Context, folderID file.FolderID
 }
 
 func (qb *ImageStore) FindByZipFileID(ctx context.Context, zipFileID file.ID) ([]*models.Image, error) {
-	table := qb.queryTable()
-	sq := dialect.From(table).Select(table.Col(idColumn)).Where(table.Col("zip_file_id").Eq(zipFileID))
+	table := qb.table()
+	fileTable := goqu.T(fileTable)
+
+	sq := dialect.From(table).
+		InnerJoin(
+			imagesFilesJoinTable,
+			goqu.On(table.Col(idColumn).Eq(imagesFilesJoinTable.Col(imageIDColumn))),
+		).
+		InnerJoin(
+			fileTable,
+			goqu.On(imagesFilesJoinTable.Col(fileIDColumn).Eq(fileTable.Col(idColumn))),
+		).
+		Select(table.Col(idColumn)).Where(
+		fileTable.Col("zip_file_id").Eq(zipFileID),
+	)
 
 	ret, err := qb.findBySubquery(ctx, sq)
 	if err != nil {
@@ -563,20 +587,21 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 
 	query.handleCriterion(ctx, criterionHandlerFunc(func(ctx context.Context, f *filterBuilder) {
 		if imageFilter.Checksum != nil {
-			f.addLeftJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
+			qb.addQueryTable(f)
+			f.addInnerJoin(fingerprintTable, "fingerprints_md5", "galleries_query.file_id = fingerprints_md5.file_id AND fingerprints_md5.type = 'md5'")
 		}
 
 		stringCriterionHandler(imageFilter.Checksum, "fingerprints_md5.fingerprint")(ctx, f)
 	}))
 	query.handleCriterion(ctx, stringCriterionHandler(imageFilter.Title, "images.title"))
 
-	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename"))
+	query.handleCriterion(ctx, pathCriterionHandler(imageFilter.Path, "images_query.parent_folder_path", "images_query.basename", qb.addQueryTable))
 	query.handleCriterion(ctx, imageFileCountCriterionHandler(qb, imageFilter.FileCount))
 	query.handleCriterion(ctx, intCriterionHandler(imageFilter.Rating, "images.rating"))
 	query.handleCriterion(ctx, intCriterionHandler(imageFilter.OCounter, "images.o_counter"))
 	query.handleCriterion(ctx, boolCriterionHandler(imageFilter.Organized, "images.organized"))
 
-	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width"))
+	query.handleCriterion(ctx, resolutionCriterionHandler(imageFilter.Resolution, "images_query.image_height", "images_query.image_width", qb.addQueryTable))
 	query.handleCriterion(ctx, imageIsMissingCriterionHandler(qb, imageFilter.IsMissing))
 
 	query.handleCriterion(ctx, imageTagsCriterionHandler(qb, imageFilter.Tags))
@@ -591,6 +616,10 @@ func (qb *ImageStore) makeFilter(ctx context.Context, imageFilter *models.ImageF
 	return query
 }
 
+func (qb *ImageStore) addQueryTable(f *filterBuilder) {
+	f.addInnerJoin(imagesQueryTable.GetTable(), "", "images.id = images_query.id")
+}
+
 func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFilterType, findFilter *models.FindFilterType) (*queryBuilder, error) {
 	if imageFilter == nil {
 		imageFilter = &models.ImageFilterType{}
@@ -602,15 +631,27 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
 	query := qb.newQuery()
 	distinctIDs(&query, imageTable)
 
-	// for convenience, join with the query view
-	query.addJoins(join{
-		table:    imagesQueryTable.GetTable(),
-		onClause: "images.id = images_query.id",
-		joinType: "INNER",
-	})
-
 	if q := findFilter.Q; q != nil && *q != "" {
-		searchColumns := []string{"images.title", "images_query.parent_folder_path", "images_query.basename", "images_query.fingerprint"}
+		query.addJoins(
+			join{
+				table:    imagesFilesTable,
+				onClause: "images_files.image_id = images.id",
+			},
+			join{
+				table:    fileTable,
+				onClause: "images_files.file_id = files.id",
+			},
+			join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			},
+			join{
+				table:    fingerprintTable,
+				onClause: "files_fingerprints.file_id = images_files.file_id",
+			},
+		)
+
+		searchColumns := []string{"images.title", "folders.path", "files.basename", "files_fingerprints.fingerprint"}
 		query.parseQueryString(searchColumns, *q)
 	}
 
@@ -621,7 +662,7 @@ func (qb *ImageStore) makeQuery(ctx context.Context, imageFilter *models.ImageFi
 
 	query.addFilter(filter)
 
-	query.sortAndPagination = qb.getImageSort(findFilter) + getPagination(findFilter)
+	qb.setImageSortAndPagination(&query, findFilter)
 
 	return &query, nil
 }
@@ -769,8 +810,10 @@ func imageTagCountCriterionHandler(qb *ImageStore, tagCount *models.IntCriterion
 
 func imageGalleriesCriterionHandler(qb *ImageStore, galleries *models.MultiCriterionInput) criterionHandlerFunc {
 	addJoinsFunc := func(f *filterBuilder) {
-		qb.galleriesRepository().join(f, "", "images.id")
-		f.addLeftJoin(galleryTable, "", "galleries_images.gallery_id = galleries.id")
+		if galleries.Modifier == models.CriterionModifierIncludes || galleries.Modifier == models.CriterionModifierIncludesAll {
+			f.addInnerJoin(galleriesImagesTable, "", "galleries_images.image_id = images.id")
+			f.addInnerJoin(galleryTable, "", "galleries_images.gallery_id = galleries.id")
+		}
 	}
 	h := qb.getMultiCriterionHandlerBuilder(galleryTable, galleriesImagesTable, galleryIDColumn, addJoinsFunc)
 
@@ -872,30 +915,54 @@ INNER JOIN (` + valuesClause + `) t ON t.column2 = pt.tag_id
 	}
 }
 
-func (qb *ImageStore) getImageSort(findFilter *models.FindFilterType) string {
-	if findFilter == nil || findFilter.Sort == nil || *findFilter.Sort == "" {
-		return ""
-	}
-	sort := findFilter.GetSort("title")
-	direction := findFilter.GetDirection()
+func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *models.FindFilterType) {
+	sortClause := ""
 
-	// translate sort field
-	if sort == "file_mod_time" {
-		sort = "mod_time"
+	if findFilter != nil && findFilter.Sort != nil && *findFilter.Sort != "" {
+		sort := findFilter.GetSort("title")
+		direction := findFilter.GetDirection()
+
+		// translate sort field
+		if sort == "file_mod_time" {
+			sort = "mod_time"
+		}
+
+		addFilesJoin := func() {
+			q.addJoins(
+				join{
+					table:    imagesFilesTable,
+					onClause: "images_files.image_id = images.id",
+				},
+				join{
+					table:    fileTable,
+					onClause: "images_files.file_id = files.id",
+				},
+			)
+		}
+
+		switch sort {
+		case "path":
+			addFilesJoin()
+			q.addJoins(join{
+				table:    folderTable,
+				onClause: "files.parent_folder_id = folders.id",
+			})
+			sortClause = " ORDER BY folders.path " + direction + ", files.basename " + direction
+		case "file_count":
+			sortClause = getCountSort(imageTable, imagesFilesTable, imageIDColumn, direction)
+		case "tag_count":
+			sortClause = getCountSort(imageTable, imagesTagsTable, imageIDColumn, direction)
+		case "performer_count":
+			sortClause = getCountSort(imageTable, performersImagesTable, imageIDColumn, direction)
+		case "mod_time", "size":
+			addFilesJoin()
+			sortClause = getSort(sort, direction, "files")
+		default:
+			sortClause = getSort(sort, direction, "images")
+		}
 	}
 
-	switch sort {
-	case "path":
-		return " ORDER BY images_query.parent_folder_path " + direction + ", images_query.basename " + direction
-	case "file_count":
-		return getCountSort(imageTable, imagesFilesTable, imageIDColumn, direction)
-	case "tag_count":
-		return getCountSort(imageTable, imagesTagsTable, imageIDColumn, direction)
-	case "performer_count":
-		return getCountSort(imageTable, performersImagesTable, imageIDColumn, direction)
-	default:
-		return getSort(sort, direction, "images_query")
-	}
+	q.sortAndPagination = sortClause + getPagination(findFilter)
 }
 
 func (qb *ImageStore) galleriesRepository() *joinRepository {