From 76e559887653b4009d06e4591919948cfa6f64b2 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 20 Feb 2024 14:04:31 +1100 Subject: [PATCH] Improve handling of moved and added video files (#4598) * If old file path is not in library, treat as move * Use existing phash if file with same oshash exists --- internal/manager/manager_tasks.go | 7 ++- internal/manager/task_generate_phash.go | 63 ++++++++++++++++++++++--- internal/manager/task_scan.go | 13 +++-- pkg/file/scan.go | 10 +++- 4 files changed, 75 insertions(+), 18 deletions(-) diff --git a/internal/manager/manager_tasks.go b/internal/manager/manager_tasks.go index 1202d0327..9a9037d9f 100644 --- a/internal/manager/manager_tasks.go +++ b/internal/manager/manager_tasks.go @@ -19,14 +19,17 @@ import ( ) func useAsVideo(pathname string) bool { - if instance.Config.IsCreateImageClipsFromVideos() && config.StashConfigs.GetStashFromDirPath(instance.Config.GetStashPaths(), pathname).ExcludeVideo { + stash := config.StashConfigs.GetStashFromDirPath(instance.Config.GetStashPaths(), pathname) + + if instance.Config.IsCreateImageClipsFromVideos() && stash != nil && stash.ExcludeVideo { return false } return isVideo(pathname) } func useAsImage(pathname string) bool { - if instance.Config.IsCreateImageClipsFromVideos() && config.StashConfigs.GetStashFromDirPath(instance.Config.GetStashPaths(), pathname).ExcludeVideo { + stash := config.StashConfigs.GetStashFromDirPath(instance.Config.GetStashPaths(), pathname) + if instance.Config.IsCreateImageClipsFromVideos() && stash != nil && stash.ExcludeVideo { return isImage(pathname) || isVideo(pathname) } return isImage(pathname) diff --git a/internal/manager/task_generate_phash.go b/internal/manager/task_generate_phash.go index ec869b128..54dc1a10b 100644 --- a/internal/manager/task_generate_phash.go +++ b/internal/manager/task_generate_phash.go @@ -25,19 +25,38 @@ func (t *GeneratePhashTask) Start(ctx context.Context) { return } - hash, err := videophash.Generate(instance.FFMpeg, t.File) - if err != nil { - logger.Errorf("error generating phash: %s", err.Error()) - logErrorOutput(err) - return + var hash int64 + set := false + + // #4393 - if there is a file with the same oshash, we can use the same phash + // only use this if we're not overwriting + if !t.Overwrite { + existing, err := t.findExistingPhash(ctx) + if err != nil { + logger.Warnf("Error finding existing phash: %v", err) + } else if existing != nil { + logger.Infof("Using existing phash for %s", t.File.Path) + hash = existing.(int64) + set = true + } + } + + if !set { + generated, err := videophash.Generate(instance.FFMpeg, t.File) + if err != nil { + logger.Errorf("Error generating phash: %v", err) + logErrorOutput(err) + return + } + + hash = int64(*generated) } r := t.repository if err := r.WithTxn(ctx, func(ctx context.Context) error { - hashValue := int64(*hash) t.File.Fingerprints = t.File.Fingerprints.AppendUnique(models.Fingerprint{ Type: models.FingerprintTypePhash, - Fingerprint: hashValue, + Fingerprint: hash, }) return r.File.Update(ctx, t.File) @@ -46,6 +65,36 @@ func (t *GeneratePhashTask) Start(ctx context.Context) { } } +func (t *GeneratePhashTask) findExistingPhash(ctx context.Context) (interface{}, error) { + r := t.repository + var ret interface{} + if err := r.WithReadTxn(ctx, func(ctx context.Context) error { + oshash := t.File.Fingerprints.Get(models.FingerprintTypeOshash) + + // find other files with the same oshash + files, err := r.File.FindByFingerprint(ctx, models.Fingerprint{ + Type: models.FingerprintTypeOshash, + Fingerprint: oshash, + }) + if err != nil { + return fmt.Errorf("finding files by oshash: %w", err) + } + + // find the first file with a phash + for _, file := range files { + if phash := file.Base().Fingerprints.Get(models.FingerprintTypePhash); phash != nil { + ret = phash + return nil + } + } + return nil + }); err != nil { + return nil, err + } + + return ret, nil +} + func (t *GeneratePhashTask) required() bool { if t.Overwrite { return true diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index 596a06143..342b5c133 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -264,6 +264,12 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) return false } + s := f.stashPaths.GetStashFromDirPath(path) + if s == nil { + logger.Debugf("Skipping %s as it is not in the stash library", path) + return false + } + isVideoFile := useAsVideo(path) isImageFile := useAsImage(path) isZipFile := fsutil.MatchExtension(path, f.zipExt) @@ -288,13 +294,6 @@ func (f *scanFilter) Accept(ctx context.Context, path string, info fs.FileInfo) return false } - s := f.stashPaths.GetStashFromDirPath(path) - - if s == nil { - logger.Debugf("Skipping %s as it is not in the stash library", path) - return false - } - // shortcut: skip the directory entirely if it matches both exclusion patterns // add a trailing separator so that it correctly matches against patterns like path/.* pathExcludeTest := path + string(filepath.Separator) diff --git a/pkg/file/scan.go b/pkg/file/scan.go index a9c20b518..e32095744 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -867,9 +867,11 @@ func (s *scanJob) handleRename(ctx context.Context, f models.File, fp []models.F continue } - if _, err := fs.Lstat(other.Base().Path); err != nil { + info, err := fs.Lstat(other.Base().Path) + switch { + case err != nil: missing = append(missing, other) - } else if strings.EqualFold(f.Base().Path, other.Base().Path) { + case strings.EqualFold(f.Base().Path, other.Base().Path): // #1426 - if file exists but is a case-insensitive match for the // original filename, and the filesystem is case-insensitive // then treat it as a move @@ -877,6 +879,10 @@ func (s *scanJob) handleRename(ctx context.Context, f models.File, fp []models.F // treat as a move missing = append(missing, other) } + case !s.acceptEntry(ctx, other.Base().Path, info): + // #4393 - if the file is no longer in the configured library paths, treat it as a move + logger.Debugf("File %q no longer in library paths. Treating as a move.", other.Base().Path) + missing = append(missing, other) } }