diff --git a/pkg/sqlite/filter.go b/pkg/sqlite/filter.go index 143487af2..fa6759ae6 100644 --- a/pkg/sqlite/filter.go +++ b/pkg/sqlite/filter.go @@ -96,6 +96,9 @@ type join struct { onClause string joinType string args []interface{} + + // if true, indicates this is required for sorting only + sort bool } // equals returns true if the other join alias/table is equal to this one @@ -127,30 +130,45 @@ func (j join) toSQL() string { type joins []join +// addUnique only adds if not already present +// returns true if added +func (j *joins) addUnique(newJoin join) bool { + found := false + for i, jj := range *j { + if jj.equals(newJoin) { + found = true + // if sort is false on the new join, but true on the existing, set the false + if !newJoin.sort && jj.sort { + (*j)[i].sort = false + } + break + } + } + + if !found { + *j = append(*j, newJoin) + } + return !found +} + func (j *joins) add(newJoins ...join) { // only add if not already joined for _, newJoin := range newJoins { - found := false - for _, jj := range *j { - if jj.equals(newJoin) { - found = true - break - } - } - - if !found { - *j = append(*j, newJoin) - } + j.addUnique(newJoin) } } -func (j *joins) toSQL() string { +func (j *joins) toSQL(includeSortPagination bool) string { if len(*j) == 0 { return "" } var ret []string for _, jj := range *j { + // skip sort-only joins if not including sort/pagination + if !includeSortPagination && jj.sort { + continue + } ret = append(ret, jj.toSQL()) } diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 9cfe38b1f..a63e84214 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -800,10 +800,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F addFileTable := func() { query.addJoins( join{ + sort: true, table: galleriesFilesTable, onClause: "galleries_files.gallery_id = galleries.id", }, join{ + sort: true, table: fileTable, onClause: "galleries_files.file_id = files.id", }, @@ -813,10 +815,12 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F addFolderTable := func() { query.addJoins( join{ + sort: true, table: folderTable, onClause: "folders.id = galleries.folder_id", }, join{ + sort: true, table: folderTable, as: "file_folder", onClause: "files.parent_folder_id = file_folder.id", diff --git a/pkg/sqlite/group.go b/pkg/sqlite/group.go index f0f8d6b40..9d0527752 100644 --- a/pkg/sqlite/group.go +++ b/pkg/sqlite/group.go @@ -518,7 +518,7 @@ func (qb *GroupStore) setGroupSort(query *queryBuilder, findFilter *models.FindF } else { // this will give unexpected results if the query is not filtered by a parent group and // the group has multiple parents and order indexes - query.join(groupRelationsTable, "", "groups.id = groups_relations.sub_id") + query.joinSort(groupRelationsTable, "", "groups.id = groups_relations.sub_id") query.sortAndPagination += getSort("order_index", direction, groupRelationsTable) } case "tag_count": diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 1588fa415..a903a62fd 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -965,10 +965,12 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod addFilesJoin := func() { q.addJoins( join{ + sort: true, table: imagesFilesTable, onClause: "images_files.image_id = images.id", }, join{ + sort: true, table: fileTable, onClause: "images_files.file_id = files.id", }, @@ -977,6 +979,7 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod addFolderJoin := func() { q.addJoins(join{ + sort: true, table: folderTable, onClause: "files.parent_folder_id = folders.id", }) diff --git a/pkg/sqlite/query.go b/pkg/sqlite/query.go index 4f4c0c8db..99c1f4e5f 100644 --- a/pkg/sqlite/query.go +++ b/pkg/sqlite/query.go @@ -24,8 +24,8 @@ type queryBuilder struct { sortAndPagination string } -func (qb queryBuilder) body() string { - return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL()) +func (qb queryBuilder) body(includeSortPagination bool) string { + return fmt.Sprintf("SELECT %s FROM %s%s", strings.Join(qb.columns, ", "), qb.from, qb.joins.toSQL(includeSortPagination)) } func (qb *queryBuilder) addColumn(column string) { @@ -33,7 +33,7 @@ func (qb *queryBuilder) addColumn(column string) { } func (qb queryBuilder) toSQL(includeSortPagination bool) string { - body := qb.body() + body := qb.body(includeSortPagination) withClause := "" if len(qb.withClauses) > 0 { @@ -59,12 +59,14 @@ func (qb queryBuilder) findIDs(ctx context.Context) ([]int, error) { } func (qb queryBuilder) executeFind(ctx context.Context) ([]int, int, error) { - body := qb.body() + const includeSortPagination = true + body := qb.body(includeSortPagination) return qb.repository.executeFindQuery(ctx, body, qb.args, qb.sortAndPagination, qb.whereClauses, qb.havingClauses, qb.withClauses, qb.recursiveWith) } func (qb queryBuilder) executeCount(ctx context.Context) (int, error) { - body := qb.body() + const includeSortPagination = false + body := qb.body(includeSortPagination) withClause := "" if len(qb.withClauses) > 0 { @@ -131,10 +133,23 @@ func (qb *queryBuilder) join(table, as, onClause string) { qb.joins.add(newJoin) } +func (qb *queryBuilder) joinSort(table, as, onClause string) { + newJoin := join{ + sort: true, + table: table, + as: as, + onClause: onClause, + joinType: "LEFT", + } + + qb.joins.add(newJoin) +} + func (qb *queryBuilder) addJoins(joins ...join) { - qb.joins.add(joins...) for _, j := range joins { - qb.args = append(qb.args, j.args...) + if qb.joins.addUnique(j) { + qb.args = append(qb.args, j.args...) + } } } diff --git a/pkg/sqlite/repository.go b/pkg/sqlite/repository.go index ac2954cfb..18d501e3a 100644 --- a/pkg/sqlite/repository.go +++ b/pkg/sqlite/repository.go @@ -96,7 +96,7 @@ func (r *repository) runIdsQuery(ctx context.Context, query string, args []inter } func (r *repository) queryFunc(ctx context.Context, query string, args []interface{}, single bool, f func(rows *sqlx.Rows) error) error { - rows, err := dbWrapper.Queryx(ctx, query, args...) + rows, err := dbWrapper.QueryxContext(ctx, query, args...) if err != nil && !errors.Is(err, sql.ErrNoRows) { return err @@ -119,13 +119,12 @@ func (r *repository) queryFunc(ctx context.Context, query string, args []interfa return nil } +// queryStruct executes a query and scans the result into the provided struct. +// Unlike the other query methods, this will return an error if no rows are found. func (r *repository) queryStruct(ctx context.Context, query string, args []interface{}, out interface{}) error { - if err := r.queryFunc(ctx, query, args, true, func(rows *sqlx.Rows) error { - if err := rows.StructScan(out); err != nil { - return err - } - return nil - }); err != nil { + // changed from queryFunc, since it was not logging the performance correctly, + // since the query doesn't actually execute until Scan is called + if err := dbWrapper.Get(ctx, out, query, args...); err != nil { return fmt.Errorf("executing query: %s [%v]: %w", query, args, err) } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index 40feb5847..955a98419 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1157,10 +1157,12 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable := func() { query.addJoins( join{ + sort: true, table: scenesFilesTable, onClause: "scenes_files.scene_id = scenes.id", }, join{ + sort: true, table: fileTable, onClause: "scenes_files.file_id = files.id", }, @@ -1171,6 +1173,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable() query.addJoins( join{ + sort: true, table: videoFileTable, onClause: "video_files.file_id = scenes_files.file_id", }, @@ -1180,6 +1183,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFolderTable := func() { query.addJoins( join{ + sort: true, table: folderTable, onClause: "files.parent_folder_id = folders.id", }, @@ -1189,10 +1193,10 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF direction := findFilter.GetDirection() switch sort { case "movie_scene_number": - query.join(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id") + query.joinSort(groupsScenesTable, "", "scenes.id = groups_scenes.scene_id") query.sortAndPagination += getSort("scene_index", direction, groupsScenesTable) case "group_scene_number": - query.join(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id") + query.joinSort(groupsScenesTable, "scene_group", "scenes.id = scene_group.scene_id") query.sortAndPagination += getSort("scene_index", direction, "scene_group") case "tag_count": query.sortAndPagination += getCountSort(sceneTable, scenesTagsTable, sceneIDColumn, direction) @@ -1210,6 +1214,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF addFileTable() query.addJoins( join{ + sort: true, table: fingerprintTable, as: "fingerprints_phash", onClause: "scenes_files.file_id = fingerprints_phash.file_id AND fingerprints_phash.type = 'phash'", @@ -1274,7 +1279,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF getSortDirection(direction), ) case "studio": - query.join(studioTable, "", "scenes.studio_id = studios.id") + query.joinSort(studioTable, "", "scenes.studio_id = studios.id") query.sortAndPagination += getSort("name", direction, studioTable) default: query.sortAndPagination += getSort(sort, direction, "scenes") diff --git a/pkg/sqlite/scene_marker.go b/pkg/sqlite/scene_marker.go index 59a4137e1..d47df0e0f 100644 --- a/pkg/sqlite/scene_marker.go +++ b/pkg/sqlite/scene_marker.go @@ -392,10 +392,10 @@ func (qb *SceneMarkerStore) setSceneMarkerSort(query *queryBuilder, findFilter * switch sort { case "scenes_updated_at": sort = "updated_at" - query.join(sceneTable, "", "scenes.id = scene_markers.scene_id") + query.joinSort(sceneTable, "", "scenes.id = scene_markers.scene_id") query.sortAndPagination += getSort(sort, direction, sceneTable) case "title": - query.join(tagTable, "", "scene_markers.primary_tag_id = tags.id") + query.joinSort(tagTable, "", "scene_markers.primary_tag_id = tags.id") query.sortAndPagination += " ORDER BY COALESCE(NULLIF(scene_markers.title,''), tags.name) COLLATE NATURAL_CI " + direction case "duration": sort = "(scene_markers.end_seconds - scene_markers.seconds)" diff --git a/pkg/sqlite/tx.go b/pkg/sqlite/tx.go index a2e272aa9..b6701dc81 100644 --- a/pkg/sqlite/tx.go +++ b/pkg/sqlite/tx.go @@ -16,8 +16,8 @@ const ( type dbReader interface { Get(dest interface{}, query string, args ...interface{}) error - Select(dest interface{}, query string, args ...interface{}) error - Queryx(query string, args ...interface{}) (*sqlx.Rows, error) + GetContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error + SelectContext(ctx context.Context, dest interface{}, query string, args ...interface{}) error QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) } @@ -54,7 +54,7 @@ func (*dbWrapperType) Get(ctx context.Context, dest interface{}, query string, a } start := time.Now() - err = tx.Get(dest, query, args...) + err = tx.GetContext(ctx, dest, query, args...) logSQL(start, query, args...) return sqlError(err, query, args...) @@ -67,7 +67,7 @@ func (*dbWrapperType) Select(ctx context.Context, dest interface{}, query string } start := time.Now() - err = tx.Select(dest, query, args...) + err = tx.SelectContext(ctx, dest, query, args...) logSQL(start, query, args...) return sqlError(err, query, args...) @@ -80,23 +80,14 @@ func (*dbWrapperType) Queryx(ctx context.Context, query string, args ...interfac } start := time.Now() - ret, err := tx.Queryx(query, args...) + ret, err := tx.QueryxContext(ctx, query, args...) logSQL(start, query, args...) return ret, sqlError(err, query, args...) } func (*dbWrapperType) QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) { - tx, err := getDBReader(ctx) - if err != nil { - return nil, sqlError(err, query, args...) - } - - start := time.Now() - ret, err := tx.QueryxContext(ctx, query, args...) - logSQL(start, query, args...) - - return ret, sqlError(err, query, args...) + return dbWrapper.Queryx(ctx, query, args...) } func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface{}) (sql.Result, error) { @@ -106,7 +97,7 @@ func (*dbWrapperType) NamedExec(ctx context.Context, query string, arg interface } start := time.Now() - ret, err := tx.NamedExec(query, arg) + ret, err := tx.NamedExecContext(ctx, query, arg) logSQL(start, query, arg) return ret, sqlError(err, query, arg) @@ -119,7 +110,7 @@ func (*dbWrapperType) Exec(ctx context.Context, query string, args ...interface{ } start := time.Now() - ret, err := tx.Exec(query, args...) + ret, err := tx.ExecContext(ctx, query, args...) logSQL(start, query, args...) return ret, sqlError(err, query, args...)