SQL performance improvements (#6378)

* Change queryStruct to use tx.Get instead of queryFunc

Using queryFunc meant that the performance logging was inaccurate due to the query actually being executed during the call to Scan.

* Only add join args if join was added

* Omit joins that are only used for sorting when skipping sorting

Should provide some marginal improvement on systems with a lot of items.

* Make all calls to the database pass context.

This means that long queries can be cancelled by navigating to another page. Previously the query would continue to run, impacting on future queries.

pkg/sqlite/filter.go

@@ -96,6 +96,9 @@ type join struct {
 	onClause string
 	joinType string
 	args     []interface{}
+
+	// if true, indicates this is required for sorting only
+	sort bool
 }
 
 // equals returns true if the other join alias/table is equal to this one
@@ -127,13 +130,17 @@ func (j join) toSQL() string {
 
 type joins []join
 
-func (j *joins) add(newJoins ...join) {
+// addUnique only adds if not already present
-	// only add if not already joined
+// returns true if added
-	for _, newJoin := range newJoins {
+func (j *joins) addUnique(newJoin join) bool {
 	found := false
-		for _, jj := range *j {
+	for i, jj := range *j {
 		if jj.equals(newJoin) {
 			found = true
+			// if sort is false on the new join, but true on the existing, set the false
+			if !newJoin.sort && jj.sort {
+				(*j)[i].sort = false
+			}
 			break
 		}
 	}
@@ -141,16 +148,27 @@ func (j *joins) add(newJoins ...join) {
 	if !found {
 		*j = append(*j, newJoin)
 	}
+	return !found
+}
+
+func (j *joins) add(newJoins ...join) {
+	// only add if not already joined
+	for _, newJoin := range newJoins {
+		j.addUnique(newJoin)
 	}
 }
 
-func (j *joins) toSQL() string {
+func (j *joins) toSQL(includeSortPagination bool) string {
 	if len(*j) == 0 {
 		return ""
 	}
 
 	var ret []string
 	for _, jj := range *j {
+		// skip sort-only joins if not including sort/pagination
+		if !includeSortPagination && jj.sort {
+			continue
+		}
 		ret = append(ret, jj.toSQL())
 	}
 

pkg/sqlite/gallery.go

@@ -800,10 +800,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F
 	addFileTable := func() {
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    galleriesFilesTable,
 				onClause: "galleries_files.gallery_id = galleries.id",
 			},
 			join{
+				sort:     true,
 				table:    fileTable,
 				onClause: "galleries_files.file_id = files.id",
 			},
@@ -813,10 +815,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F
 	addFolderTable := func() {
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    folderTable,
 				onClause: "folders.id = galleries.folder_id",
 			},
 			join{
+				sort:     true,
 				table:    folderTable,
 				as:       "file_folder",
 				onClause: "files.parent_folder_id = file_folder.id",

pkg/sqlite/group.go

@@ -518,7 +518,7 @@ func (qb *GroupStore) setGroupSort(query *queryBuilder, findFilter *models.FindF
 		} else {
 			// this will give unexpected results if the query is not filtered by a parent group and
 			// the group has multiple parents and order indexes
-			query.join(groupRelationsTable, "", "groups.id = groups_relations.sub_id")
+			query.joinSort(groupRelationsTable, "", "groups.id = groups_relations.sub_id")
 			query.sortAndPagination += getSort("order_index", direction, groupRelationsTable)
 		}
 	case "tag_count":

pkg/sqlite/image.go

@@ -965,10 +965,12 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod
 		addFilesJoin := func() {
 			q.addJoins(
 				join{
+					sort:     true,
 					table:    imagesFilesTable,
 					onClause: "images_files.image_id = images.id",
 				},
 				join{
+					sort:     true,
 					table:    fileTable,
 					onClause: "images_files.file_id = files.id",
 				},
@@ -977,6 +979,7 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod
 
 		addFolderJoin := func() {
 			q.addJoins(join{
+				sort:     true,
 				table:    folderTable,
 				onClause: "files.parent_folder_id = folders.id",
 			})

pkg/sqlite/query.go

@@ -24,8 +24,8 @@ type queryBuilder struct {
 	sortAndPagination string
 }
 
-func (qb queryBuilder) body() string {
+func (qb queryBuilder) body(includeSortPagination bool) string {
-	return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL())
+	return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL(includeSortPagination))
 }
 
 func (qb *queryBuilder) addColumn(column string) {
@@ -33,7 +33,7 @@ func (qb *queryBuilder) addColumn(column string) {
 }
 
 func (qb queryBuilder) toSQL(includeSortPagination bool) string {
-	body := qb.body()
+	body := qb.body(includeSortPagination)
 
 	withClause := ""
 	if len(qb.withClauses) > 0 {
@@ -59,12 +59,14 @@ func (qb queryBuilder) findIDs(ctx context.Context) ([]int, error) {
 }
 
 func (qb queryBuilder) executeFind(ctx context.Context) ([]int, int, error) {
-	body := qb.body()
+	const includeSortPagination = true
+	body := qb.body(includeSortPagination)
 	return qb.repository.executeFindQuery(ctx, body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses, qb.withClauses, qb.recursiveWith)
 }
 
 func (qb queryBuilder) executeCount(ctx context.Context) (int, error) {
-	body := qb.body()
+	const includeSortPagination = false
+	body := qb.body(includeSortPagination)
 
 	withClause := ""
 	if len(qb.withClauses) > 0 {
@@ -131,12 +133,25 @@ func (qb *queryBuilder) join(table, as, onClause string) {
 	qb.joins.add(newJoin)
 }
 
+func (qb *queryBuilder) joinSort(table, as, onClause string) {
+	newJoin := join{
+		sort:     true,
+		table:    table,
+		as:       as,
+		onClause: onClause,
+		joinType: "LEFT",
+	}
+
+	qb.joins.add(newJoin)
+}
+
 func (qb *queryBuilder) addJoins(joins ...join) {
-	qb.joins.add(joins...)
 	for _, j := range joins {
+		if qb.joins.addUnique(j) {
 			qb.args = append(qb.args, j.args...)
 		}
 	}
+}
 
 func (qb *queryBuilder) addFilter(f *filterBuilder) error {
 	err := f.getError()

pkg/sqlite/repository.go

@@ -96,7 +96,7 @@ func (r *repository) runIdsQuery(ctx context.Context, query string, args []inter
 }
 
 func (r *repository) queryFunc(ctx context.Context, query string, args []interface{}, single bool, f func(rows *sqlx.Rows) error) error {
-	rows, err := dbWrapper.Queryx(ctx, query, args...)
+	rows, err := dbWrapper.QueryxContext(ctx, query, args...)
 
 	if err != nil && !errors.Is(err, sql.ErrNoRows) {
 		return err
@@ -119,13 +119,12 @@ func (r *repository) queryFunc(ctx context.Context, query string, args []interfa
 	return nil
 }
 
+// queryStruct executes a query and scans the result into the provided struct.
+// Unlike the other query methods, this will return an error if no rows are found.
 func (r *repository) queryStruct(ctx context.Context, query string, args []interface{}, out interface{}) error {
-	if err := r.queryFunc(ctx, query, args, true, func(rows *sqlx.Rows) error {
+	// changed from queryFunc, since it was not logging the performance correctly,
-		if err := rows.StructScan(out); err != nil {
+	// since the query doesn't actually execute until Scan is called
-			return err
+	if err := dbWrapper.Get(ctx, out, query, args...); err != nil {
-		}
-		return nil
-	}); err != nil {
 		return fmt.Errorf("executing query: %s [%v]: %w", query, args, err)
 	}
 

pkg/sqlite/scene.go

@@ -1157,10 +1157,12 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 	addFileTable := func() {
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    scenesFilesTable,
 				onClause: "scenes_files.scene_id = scenes.id",
 			},
 			join{
+				sort:     true,
 				table:    fileTable,
 				onClause: "scenes_files.file_id = files.id",
 			},
@@ -1171,6 +1173,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 		addFileTable()
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    videoFileTable,
 				onClause: "video_files.file_id = scenes_files.file_id",
 			},
@@ -1180,6 +1183,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 	addFolderTable := func() {
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    folderTable,
 				onClause: "files.parent_folder_id = folders.id",
 			},
@@ -1189,10 +1193,10 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 	direction := findFilter.GetDirection()
 	switch sort {
 	case "movie_scene_number":
-		query.join(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id")
+		query.joinSort(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id")
 		query.sortAndPagination += getSort("scene_index", direction, groupsScenesTable)
 	case "group_scene_number":
-		query.join(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id")
+		query.joinSort(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id")
 		query.sortAndPagination += getSort("scene_index", direction, "scene_group")
 	case "tag_count":
 		query.sortAndPagination += getCountSort(sceneTable, scenesTagsTable, sceneIDColumn, direction)
@@ -1210,6 +1214,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 		addFileTable()
 		query.addJoins(
 			join{
+				sort:     true,
 				table:    fingerprintTable,
 				as:       "fingerprints_phash",
 				onClause: "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'",
@@ -1274,7 +1279,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF
 			getSortDirection(direction),
 		)
 	case "studio":
-		query.join(studioTable, "", "scenes.studio_id = studios.id")
+		query.joinSort(studioTable, "", "scenes.studio_id = studios.id")
 		query.sortAndPagination += getSort("name", direction, studioTable)
 	default:
 		query.sortAndPagination += getSort(sort, direction, "scenes")

pkg/sqlite/scene_marker.go

@@ -392,10 +392,10 @@ func (qb *SceneMarkerStore) setSceneMarkerSort(query *queryBuilder, findFilter *
 	switch sort {
 	case "scenes_updated_at":
 		sort = "updated_at"
-		query.join(sceneTable, "", "scenes.id = scene_markers.scene_id")
+		query.joinSort(sceneTable, "", "scenes.id = scene_markers.scene_id")
 		query.sortAndPagination += getSort(sort, direction, sceneTable)
 	case "title":
-		query.join(tagTable, "", "scene_markers.primary_tag_id = tags.id")
+		query.joinSort(tagTable, "", "scene_markers.primary_tag_id = tags.id")
 		query.sortAndPagination += " ORDER BY COALESCE(NULLIF(scene_markers.title,''), tags.name) COLLATE NATURAL_CI " + direction
 	case "duration":
 		sort = "(scene_markers.end_seconds - scene_markers.seconds)"

pkg/sqlite/tx.go

@@ -16,8 +16,8 @@ const (
 
 type dbReader interface {
 	Get(dest interface{}, query string, args ...interface{}) error
-	Select(dest interface{}, query string, args ...interface{}) error
+	GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error
-	Queryx(query string, args ...interface{}) (*sqlx.Rows, error)
+	SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error
 	QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error)
 }
 
@@ -54,7 +54,7 @@ func (*dbWrapperType) Get(ctx context.Context, dest interface{}, query string, a
 	}
 
 	start := time.Now()
-	err = tx.Get(dest, query, args...)
+	err = tx.GetContext(ctx, dest, query, args...)
 	logSQL(start, query, args...)
 
 	return sqlError(err, query, args...)
@@ -67,7 +67,7 @@ func (*dbWrapperType) Select(ctx context.Context, dest interface{}, query string
 	}
 
 	start := time.Now()
-	err = tx.Select(dest, query, args...)
+	err = tx.SelectContext(ctx, dest, query, args...)
 	logSQL(start, query, args...)
 
 	return sqlError(err, query, args...)
@@ -80,23 +80,14 @@ func (*dbWrapperType) Queryx(ctx context.Context, query string, args ...interfac
 	}
 
 	start := time.Now()
-	ret, err := tx.Queryx(query, args...)
+	ret, err := tx.QueryxContext(ctx, query, args...)
 	logSQL(start, query, args...)
 
 	return ret, sqlError(err, query, args...)
 }
 
 func (*dbWrapperType) QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) {
-	tx, err := getDBReader(ctx)
+	return dbWrapper.Queryx(ctx, query, args...)
-	if err != nil {
-		return nil, sqlError(err, query, args...)
-	}
-
-	start := time.Now()
-	ret, err := tx.QueryxContext(ctx, query, args...)
-	logSQL(start, query, args...)
-
-	return ret, sqlError(err, query, args...)
 }
 
 func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface{}) (sql.Result, error) {
@@ -106,7 +97,7 @@ func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface
 	}
 
 	start := time.Now()
-	ret, err := tx.NamedExec(query, arg)
+	ret, err := tx.NamedExecContext(ctx, query, arg)
 	logSQL(start, query, arg)
 
 	return ret, sqlError(err, query, arg)
@@ -119,7 +110,7 @@ func (*dbWrapperType) Exec(ctx context.Context, query string, args ...interface{
 	}
 
 	start := time.Now()
-	ret, err := tx.Exec(query, args...)
+	ret, err := tx.ExecContext(ctx, query, args...)
 	logSQL(start, query, args...)
 
 	return ret, sqlError(err, query, args...)