From 0fd7a2ac20f9bac221369a095953c2a265e77709 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:08:31 +1100 Subject: [PATCH] SQL performance improvements (#6378) * Change queryStruct to use tx.Get instead of queryFunc Using queryFunc meant that the performance logging was inaccurate due to the query actually being executed during the call to Scan. * Only add join args if join was added * Omit joins that are only used for sorting when skipping sorting Should provide some marginal improvement on systems with a lot of items. * Make all calls to the database pass context. This means that long queries can be cancelled by navigating to another page. Previously the query would continue to run, impacting on future queries. --- pkg/sqlite/filter.go | 42 +++++++++++++++++++++++++++----------- pkg/sqlite/gallery.go | 4 ++++ pkg/sqlite/group.go | 2 +- pkg/sqlite/image.go | 3 +++ pkg/sqlite/query.go | 29 +++++++++++++++++++------- pkg/sqlite/repository.go | 13 ++++++------ pkg/sqlite/scene.go | 11 +++++++--- pkg/sqlite/scene_marker.go | 4 ++-- pkg/sqlite/tx.go | 25 ++++++++--------------- 9 files changed, 84 insertions(+), 49 deletions(-) diff --git a/pkg/sqlite/filter.go b/pkg/sqlite/filter.go index 143487af2..fa6759ae6 100644 --- a/pkg/sqlite/filter.go +++ b/pkg/sqlite/filter.go @@ -96,6 +96,9 @@ type join struct { onClause string joinType string args []interface{} + + // if true, indicates this is required for sorting only + sort bool } // equals returns true if the other join alias/table is equal to this one @@ -127,30 +130,45 @@ func (j join) toSQL() string { type joins []join +// addUnique only adds if not already present +// returns true if added +func (j *joins) addUnique(newJoin join) bool { + found := false + for i, jj := range *j { + if jj.equals(newJoin) { + found = true + // if sort is false on the new join, but true on the existing, set the false + if !newJoin.sort && jj.sort { + (*j)[i].sort = false + } + break + } + } + + if !found { + *j = append(*j, newJoin) + } + return !found +} + func (j *joins) add(newJoins ...join) { // only add if not already joined for _, newJoin := range newJoins { - found := false - for _, jj := range *j { - if jj.equals(newJoin) { - found = true - break - } - } - - if !found { - *j = append(*j, newJoin) - } + j.addUnique(newJoin) } } -func (j *joins) toSQL() string { +func (j *joins) toSQL(includeSortPagination bool) string { if len(*j) == 0 { return "" } var ret []string for _, jj := range *j { + // skip sort-only joins if not including sort/pagination + if !includeSortPagination && jj.sort { + continue + } ret = append(ret, jj.toSQL()) } diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 9cfe38b1f..a63e84214 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -800,10 +800,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F addFileTable := func() { query.addJoins( join{ + sort: true, table: galleriesFilesTable, onClause: "galleries_files.gallery_id = galleries.id", }, join{ + sort: true, table: fileTable, onClause: "galleries_files.file_id = files.id", }, @@ -813,10 +815,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F addFolderTable := func() { query.addJoins( join{ + sort: true, table: folderTable, onClause: "folders.id = galleries.folder_id", }, join{ + sort: true, table: folderTable, as: "file_folder", onClause: "files.parent_folder_id = file_folder.id", diff --git a/pkg/sqlite/group.go b/pkg/sqlite/group.go index f0f8d6b40..9d0527752 100644 --- a/pkg/sqlite/group.go +++ b/pkg/sqlite/group.go @@ -518,7 +518,7 @@ func (qb *GroupStore) setGroupSort(query *queryBuilder, findFilter *models.FindF } else { // this will give unexpected results if the query is not filtered by a parent group and // the group has multiple parents and order indexes - query.join(groupRelationsTable, "", "groups.id = groups_relations.sub_id") + query.joinSort(groupRelationsTable, "", "groups.id = groups_relations.sub_id") query.sortAndPagination += getSort("order_index", direction, groupRelationsTable) } case "tag_count": diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 1588fa415..a903a62fd 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -965,10 +965,12 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod addFilesJoin := func() { q.addJoins( join{ + sort: true, table: imagesFilesTable, onClause: "images_files.image_id = images.id", }, join{ + sort: true, table: fileTable, onClause: "images_files.file_id = files.id", }, @@ -977,6 +979,7 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod addFolderJoin := func() { q.addJoins(join{ + sort: true, table: folderTable, onClause: "files.parent_folder_id = folders.id", }) diff --git a/pkg/sqlite/query.go b/pkg/sqlite/query.go index 4f4c0c8db..99c1f4e5f 100644 --- a/pkg/sqlite/query.go +++ b/pkg/sqlite/query.go @@ -24,8 +24,8 @@ type queryBuilder struct { sortAndPagination string } -func (qb queryBuilder) body() string { - return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL()) +func (qb queryBuilder) body(includeSortPagination bool) string { + return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL(includeSortPagination)) } func (qb *queryBuilder) addColumn(column string) { @@ -33,7 +33,7 @@ func (qb *queryBuilder) addColumn(column string) { } func (qb queryBuilder) toSQL(includeSortPagination bool) string { - body := qb.body() + body := qb.body(includeSortPagination) withClause := "" if len(qb.withClauses) > 0 { @@ -59,12 +59,14 @@ func (qb queryBuilder) findIDs(ctx context.Context) ([]int, error) { } func (qb queryBuilder) executeFind(ctx context.Context) ([]int, int, error) { - body := qb.body() + const includeSortPagination = true + body := qb.body(includeSortPagination) return qb.repository.executeFindQuery(ctx, body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses, qb.withClauses, qb.recursiveWith) } func (qb queryBuilder) executeCount(ctx context.Context) (int, error) { - body := qb.body() + const includeSortPagination = false + body := qb.body(includeSortPagination) withClause := "" if len(qb.withClauses) > 0 { @@ -131,10 +133,23 @@ func (qb *queryBuilder) join(table, as, onClause string) { qb.joins.add(newJoin) } +func (qb *queryBuilder) joinSort(table, as, onClause string) { + newJoin := join{ + sort: true, + table: table, + as: as, + onClause: onClause, + joinType: "LEFT", + } + + qb.joins.add(newJoin) +} + func (qb *queryBuilder) addJoins(joins ...join) { - qb.joins.add(joins...) for _, j := range joins { - qb.args = append(qb.args, j.args...) + if qb.joins.addUnique(j) { + qb.args = append(qb.args, j.args...) + } } } diff --git a/pkg/sqlite/repository.go b/pkg/sqlite/repository.go index ac2954cfb..18d501e3a 100644 --- a/pkg/sqlite/repository.go +++ b/pkg/sqlite/repository.go @@ -96,7 +96,7 @@ func (r *repository) runIdsQuery(ctx context.Context, query string, args []inter } func (r *repository) queryFunc(ctx context.Context, query string, args []interface{}, single bool, f func(rows *sqlx.Rows) error) error { - rows, err := dbWrapper.Queryx(ctx, query, args...) + rows, err := dbWrapper.QueryxContext(ctx, query, args...) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err @@ -119,13 +119,12 @@ func (r *repository) queryFunc(ctx context.Context, query string, args []interfa return nil } +// queryStruct executes a query and scans the result into the provided struct. +// Unlike the other query methods, this will return an error if no rows are found. func (r *repository) queryStruct(ctx context.Context, query string, args []interface{}, out interface{}) error { - if err := r.queryFunc(ctx, query, args, true, func(rows *sqlx.Rows) error { - if err := rows.StructScan(out); err != nil { - return err - } - return nil - }); err != nil { + // changed from queryFunc, since it was not logging the performance correctly, + // since the query doesn't actually execute until Scan is called + if err := dbWrapper.Get(ctx, out, query, args...); err != nil { return fmt.Errorf("executing query: %s [%v]: %w", query, args, err) } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 40feb5847..955a98419 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1157,10 +1157,12 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable := func() { query.addJoins( join{ + sort: true, table: scenesFilesTable, onClause: "scenes_files.scene_id = scenes.id", }, join{ + sort: true, table: fileTable, onClause: "scenes_files.file_id = files.id", }, @@ -1171,6 +1173,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable() query.addJoins( join{ + sort: true, table: videoFileTable, onClause: "video_files.file_id = scenes_files.file_id", }, @@ -1180,6 +1183,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFolderTable := func() { query.addJoins( join{ + sort: true, table: folderTable, onClause: "files.parent_folder_id = folders.id", }, @@ -1189,10 +1193,10 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF direction := findFilter.GetDirection() switch sort { case "movie_scene_number": - query.join(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id") + query.joinSort(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id") query.sortAndPagination += getSort("scene_index", direction, groupsScenesTable) case "group_scene_number": - query.join(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id") + query.joinSort(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id") query.sortAndPagination += getSort("scene_index", direction, "scene_group") case "tag_count": query.sortAndPagination += getCountSort(sceneTable, scenesTagsTable, sceneIDColumn, direction) @@ -1210,6 +1214,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable() query.addJoins( join{ + sort: true, table: fingerprintTable, as: "fingerprints_phash", onClause: "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'", @@ -1274,7 +1279,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF getSortDirection(direction), ) case "studio": - query.join(studioTable, "", "scenes.studio_id = studios.id") + query.joinSort(studioTable, "", "scenes.studio_id = studios.id") query.sortAndPagination += getSort("name", direction, studioTable) default: query.sortAndPagination += getSort(sort, direction, "scenes") diff --git a/pkg/sqlite/scene_marker.go b/pkg/sqlite/scene_marker.go index 59a4137e1..d47df0e0f 100644 --- a/pkg/sqlite/scene_marker.go +++ b/pkg/sqlite/scene_marker.go @@ -392,10 +392,10 @@ func (qb *SceneMarkerStore) setSceneMarkerSort(query *queryBuilder, findFilter * switch sort { case "scenes_updated_at": sort = "updated_at" - query.join(sceneTable, "", "scenes.id = scene_markers.scene_id") + query.joinSort(sceneTable, "", "scenes.id = scene_markers.scene_id") query.sortAndPagination += getSort(sort, direction, sceneTable) case "title": - query.join(tagTable, "", "scene_markers.primary_tag_id = tags.id") + query.joinSort(tagTable, "", "scene_markers.primary_tag_id = tags.id") query.sortAndPagination += " ORDER BY COALESCE(NULLIF(scene_markers.title,''), tags.name) COLLATE NATURAL_CI " + direction case "duration": sort = "(scene_markers.end_seconds - scene_markers.seconds)" diff --git a/pkg/sqlite/tx.go b/pkg/sqlite/tx.go index a2e272aa9..b6701dc81 100644 --- a/pkg/sqlite/tx.go +++ b/pkg/sqlite/tx.go @@ -16,8 +16,8 @@ const ( type dbReader interface { Get(dest interface{}, query string, args ...interface{}) error - Select(dest interface{}, query string, args ...interface{}) error - Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) } @@ -54,7 +54,7 @@ func (*dbWrapperType) Get(ctx context.Context, dest interface{}, query string, a } start := time.Now() - err = tx.Get(dest, query, args...) + err = tx.GetContext(ctx, dest, query, args...) logSQL(start, query, args...) return sqlError(err, query, args...) @@ -67,7 +67,7 @@ func (*dbWrapperType) Select(ctx context.Context, dest interface{}, query string } start := time.Now() - err = tx.Select(dest, query, args...) + err = tx.SelectContext(ctx, dest, query, args...) logSQL(start, query, args...) return sqlError(err, query, args...) @@ -80,23 +80,14 @@ func (*dbWrapperType) Queryx(ctx context.Context, query string, args ...interfac } start := time.Now() - ret, err := tx.Queryx(query, args...) + ret, err := tx.QueryxContext(ctx, query, args...) logSQL(start, query, args...) return ret, sqlError(err, query, args...) } func (*dbWrapperType) QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { - tx, err := getDBReader(ctx) - if err != nil { - return nil, sqlError(err, query, args...) - } - - start := time.Now() - ret, err := tx.QueryxContext(ctx, query, args...) - logSQL(start, query, args...) - - return ret, sqlError(err, query, args...) + return dbWrapper.Queryx(ctx, query, args...) } func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface{}) (sql.Result, error) { @@ -106,7 +97,7 @@ func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface } start := time.Now() - ret, err := tx.NamedExec(query, arg) + ret, err := tx.NamedExecContext(ctx, query, arg) logSQL(start, query, arg) return ret, sqlError(err, query, arg) @@ -119,7 +110,7 @@ func (*dbWrapperType) Exec(ctx context.Context, query string, args ...interface{ } start := time.Now() - ret, err := tx.Exec(query, args...) + ret, err := tx.ExecContext(ctx, query, args...) logSQL(start, query, args...) return ret, sqlError(err, query, args...)