From 0848b02e9384f6163b8d21cf36923859cbfdcb6e Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Sun, 25 Sep 2022 12:07:55 +1000 Subject: [PATCH] Various bug fixes (#2945) * Only update fingerprints if changed * Fix panic when loading primary file fails * Fix gallery/scene association * Fix display of scene gallery in card * Use natural_cs collation with paths for title sorting --- graphql/documents/data/scene-slim.graphql | 4 +- internal/api/routes_image.go | 6 +- internal/api/routes_scene.go | 6 +- pkg/file/fingerprint.go | 13 +++++ pkg/file/fingerprint_test.go | 71 +++++++++++++++++++++++ pkg/file/scan.go | 2 +- pkg/gallery/scan.go | 3 +- pkg/sqlite/gallery.go | 30 ++++++---- pkg/sqlite/image.go | 14 ++++- pkg/sqlite/scene.go | 20 +++++-- pkg/sqlite/sql.go | 2 +- 11 files changed, 145 insertions(+), 26 deletions(-) diff --git a/graphql/documents/data/scene-slim.graphql b/graphql/documents/data/scene-slim.graphql index a199fde90..36b903409 100644 --- a/graphql/documents/data/scene-slim.graphql +++ b/graphql/documents/data/scene-slim.graphql @@ -35,7 +35,9 @@ fragment SlimSceneData on Scene { galleries { id - path + files { + path + } title } diff --git a/internal/api/routes_image.go b/internal/api/routes_image.go index 4d37e862f..743cb7038 100644 --- a/internal/api/routes_image.go +++ b/internal/api/routes_image.go @@ -127,7 +127,11 @@ func (rs imageRoutes) ImageCtx(next http.Handler) http.Handler { } if image != nil { - _ = image.LoadPrimaryFile(ctx, rs.fileFinder) + if err := image.LoadPrimaryFile(ctx, rs.fileFinder); err != nil { + logger.Errorf("error loading primary file for image %d: %v", imageID, err) + // set image to nil so that it doesn't try to use the primary file + image = nil + } } return nil diff --git a/internal/api/routes_scene.go b/internal/api/routes_scene.go index 586325872..e058cc360 100644 --- a/internal/api/routes_scene.go +++ b/internal/api/routes_scene.go @@ -544,7 +544,11 @@ func (rs sceneRoutes) SceneCtx(next http.Handler) http.Handler { } if scene != nil { - _ = scene.LoadPrimaryFile(ctx, rs.fileFinder) + if err := scene.LoadPrimaryFile(ctx, rs.fileFinder); err != nil { + logger.Errorf("error loading primary file for scene %d: %v", sceneID, err) + // set scene to nil so that it doesn't try to use the primary file + scene = nil + } } return nil diff --git a/pkg/file/fingerprint.go b/pkg/file/fingerprint.go index b5a8f9b5b..3155276c5 100644 --- a/pkg/file/fingerprint.go +++ b/pkg/file/fingerprint.go @@ -26,6 +26,7 @@ func (f *Fingerprints) Remove(type_ string) { *f = ret } +// Equals returns true if the contents of this slice are equal to those in the other slice. func (f Fingerprints) Equals(other Fingerprints) bool { if len(f) != len(other) { return false @@ -48,6 +49,18 @@ func (f Fingerprints) Equals(other Fingerprints) bool { return true } +// ContentsChanged returns true if this Fingerprints slice contains any Fingerprints that different Fingerprint values for the matching type in other, or if this slice contains any Fingerprint types that are not in other. +func (f Fingerprints) ContentsChanged(other Fingerprints) bool { + for _, ff := range f { + oo := other.For(ff.Type) + if oo == nil || oo.Fingerprint != ff.Fingerprint { + return true + } + } + + return false +} + // For returns a pointer to the first Fingerprint element matching the provided type. func (f Fingerprints) For(type_ string) *Fingerprint { for _, fp := range f { diff --git a/pkg/file/fingerprint_test.go b/pkg/file/fingerprint_test.go index d9c992cf6..f13ce2254 100644 --- a/pkg/file/fingerprint_test.go +++ b/pkg/file/fingerprint_test.go @@ -84,3 +84,74 @@ func TestFingerprints_Equals(t *testing.T) { }) } } + +func TestFingerprints_ContentsChanged(t *testing.T) { + var ( + value1 = 1 + value2 = "2" + value3 = 1.23 + + fingerprint1 = Fingerprint{ + Type: FingerprintTypeMD5, + Fingerprint: value1, + } + fingerprint2 = Fingerprint{ + Type: FingerprintTypeOshash, + Fingerprint: value2, + } + fingerprint3 = Fingerprint{ + Type: FingerprintTypeMD5, + Fingerprint: value3, + } + ) + + tests := []struct { + name string + f Fingerprints + other Fingerprints + want bool + }{ + { + "identical", + Fingerprints{ + fingerprint1, + fingerprint2, + }, + Fingerprints{ + fingerprint1, + fingerprint2, + }, + false, + }, + { + "has new", + Fingerprints{ + fingerprint1, + fingerprint2, + }, + Fingerprints{ + fingerprint1, + }, + true, + }, + { + "has different value", + Fingerprints{ + fingerprint3, + fingerprint2, + }, + Fingerprints{ + fingerprint1, + fingerprint2, + }, + true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.f.ContentsChanged(tt.other); got != tt.want { + t.Errorf("Fingerprints.ContentsChanged() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/file/scan.go b/pkg/file/scan.go index ceb1f54dc..84c677f9a 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -859,7 +859,7 @@ func (s *scanJob) setMissingFingerprints(ctx context.Context, f scanFile, existi return nil, err } - if !fp.Equals(existing.Base().Fingerprints) { + if fp.ContentsChanged(existing.Base().Fingerprints) { existing.SetFingerprints(fp) if err := s.withTxn(ctx, func(ctx context.Context) error { diff --git a/pkg/gallery/scan.go b/pkg/gallery/scan.go index c594cc953..ed4ce0966 100644 --- a/pkg/gallery/scan.go +++ b/pkg/gallery/scan.go @@ -119,7 +119,7 @@ func (h *ScanHandler) associateScene(ctx context.Context, existing []*models.Gal } path := f.Base().Path - withoutExt := strings.TrimSuffix(path, filepath.Ext(path)) + withoutExt := strings.TrimSuffix(path, filepath.Ext(path)) + ".*" // find scenes with a file that matches scenes, err := h.SceneFinderUpdater.FindByPath(ctx, withoutExt) @@ -129,6 +129,7 @@ func (h *ScanHandler) associateScene(ctx context.Context, existing []*models.Gal for _, scene := range scenes { // found related Scene + logger.Infof("associate: Gallery %s is related to scene: %d", path, scene.ID) if err := h.SceneFinderUpdater.AddGalleryIDs(ctx, scene.ID, galleryIDs); err != nil { return err } diff --git a/pkg/sqlite/gallery.go b/pkg/sqlite/gallery.go index 7602b64c2..b00e3c444 100644 --- a/pkg/sqlite/gallery.go +++ b/pkg/sqlite/gallery.go @@ -1065,6 +1065,20 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F ) } + addFolderTable := func() { + query.addJoins( + join{ + table: folderTable, + onClause: "folders.id = galleries.folder_id", + }, + join{ + table: folderTable, + as: "file_folder", + onClause: "files.parent_folder_id = file_folder.id", + }, + ) + } + switch sort { case "file_count": query.sortAndPagination += getCountSort(galleryTable, galleriesFilesTable, galleryIDColumn, direction) @@ -1077,22 +1091,16 @@ func (qb *GalleryStore) setGallerySort(query *queryBuilder, findFilter *models.F case "path": // special handling for path addFileTable() - query.addJoins( - join{ - table: folderTable, - onClause: "folders.id = galleries.folder_id", - }, - join{ - table: folderTable, - as: "file_folder", - onClause: "files.parent_folder_id = file_folder.id", - }, - ) + addFolderTable() query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, file_folder.path %[1]s, files.basename %[1]s", direction) case "file_mod_time": sort = "mod_time" addFileTable() query.sortAndPagination += getSort(sort, direction, fileTable) + case "title": + addFileTable() + addFolderTable() + query.sortAndPagination += " ORDER BY galleries.title COLLATE NATURAL_CS " + direction + ", folders.path " + direction + ", file_folder.path " + direction + ", files.basename COLLATE NATURAL_CS " + direction default: query.sortAndPagination += getSort(sort, direction, "galleries") } diff --git a/pkg/sqlite/image.go b/pkg/sqlite/image.go index 7588d091d..5ad4f1724 100644 --- a/pkg/sqlite/image.go +++ b/pkg/sqlite/image.go @@ -989,13 +989,17 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod ) } - switch sort { - case "path": - addFilesJoin() + addFolderJoin := func() { q.addJoins(join{ table: folderTable, onClause: "files.parent_folder_id = folders.id", }) + } + + switch sort { + case "path": + addFilesJoin() + addFolderJoin() sortClause = " ORDER BY folders.path " + direction + ", files.basename " + direction case "file_count": sortClause = getCountSort(imageTable, imagesFilesTable, imageIDColumn, direction) @@ -1006,6 +1010,10 @@ func (qb *ImageStore) setImageSortAndPagination(q *queryBuilder, findFilter *mod case "mod_time", "filesize": addFilesJoin() sortClause = getSort(sort, direction, "files") + case "title": + addFilesJoin() + addFolderJoin() + sortClause = " ORDER BY images.title COLLATE NATURAL_CS " + direction + ", folders.path " + direction + ", files.basename COLLATE NATURAL_CS " + direction default: sortClause = getSort(sort, direction, "images") } diff --git a/pkg/sqlite/scene.go b/pkg/sqlite/scene.go index e9037148b..7b1fa1353 100644 --- a/pkg/sqlite/scene.go +++ b/pkg/sqlite/scene.go @@ -1324,6 +1324,15 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF ) } + addFolderTable := func() { + query.addJoins( + join{ + table: folderTable, + onClause: "files.parent_folder_id = folders.id", + }, + ) + } + direction := findFilter.GetDirection() switch sort { case "movie_scene_number": @@ -1338,12 +1347,7 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF case "path": // special handling for path addFileTable() - query.addJoins( - join{ - table: folderTable, - onClause: "files.parent_folder_id = folders.id", - }, - ) + addFolderTable() query.sortAndPagination += fmt.Sprintf(" ORDER BY folders.path %s, files.basename %[1]s", direction) case "perceptual_similarity": // special handling for phash @@ -1378,6 +1382,10 @@ func (qb *SceneStore) setSceneSort(query *queryBuilder, findFilter *models.FindF case "interactive", "interactive_speed": addVideoFileTable() query.sortAndPagination += getSort(sort, direction, videoFileTable) + case "title": + addFileTable() + addFolderTable() + query.sortAndPagination += " ORDER BY scenes.title COLLATE NATURAL_CS " + direction + ", folders.path " + direction + ", files.basename COLLATE NATURAL_CS " + direction default: query.sortAndPagination += getSort(sort, direction, "scenes") } diff --git a/pkg/sqlite/sql.go b/pkg/sqlite/sql.go index 44920903e..e80cceef8 100644 --- a/pkg/sqlite/sql.go +++ b/pkg/sqlite/sql.go @@ -65,7 +65,7 @@ func getSort(sort string, direction string, tableName string) string { return " ORDER BY COUNT(distinct " + colName + ") " + direction case strings.Compare(sort, "filesize") == 0: colName := getColumn(tableName, "size") - return " ORDER BY cast(" + colName + " as integer) " + direction + return " ORDER BY " + colName + " " + direction case strings.HasPrefix(sort, randomSeedPrefix): // seed as a parameter from the UI // turn the provided seed into a float