diff --git a/internal/manager/task_clean.go b/internal/manager/task_clean.go index 1fa0a61e3..f6d7304f8 100644 --- a/internal/manager/task_clean.go +++ b/internal/manager/task_clean.go @@ -16,6 +16,7 @@ import ( "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/scene" + "github.com/stashapp/stash/pkg/txn" ) type cleaner interface { @@ -49,11 +50,95 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) { return } + j.cleanEmptyGalleries(ctx) + j.scanSubs.notify() elapsed := time.Since(start) logger.Info(fmt.Sprintf("Finished Cleaning (%s)", elapsed)) } +func (j *cleanJob) cleanEmptyGalleries(ctx context.Context) { + const batchSize = 1000 + var toClean []int + findFilter := models.BatchFindFilter(batchSize) + if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error { + found := true + for found { + emptyGalleries, _, err := j.txnManager.Gallery.Query(ctx, &models.GalleryFilterType{ + ImageCount: &models.IntCriterionInput{ + Value: 0, + Modifier: models.CriterionModifierEquals, + }, + }, findFilter) + + if err != nil { + return err + } + + found = len(emptyGalleries) > 0 + + for _, g := range emptyGalleries { + if g.Path == "" { + continue + } + + if len(j.input.Paths) > 0 && !fsutil.IsPathInDirs(j.input.Paths, g.Path) { + continue + } + + logger.Infof("Gallery has 0 images. Marking to clean: %s", g.DisplayName()) + toClean = append(toClean, g.ID) + } + + *findFilter.Page++ + } + + return nil + }); err != nil { + logger.Errorf("Error finding empty galleries: %v", err) + return + } + + if !j.input.DryRun { + for _, id := range toClean { + j.deleteGallery(ctx, id) + } + } +} + +func (j *cleanJob) deleteGallery(ctx context.Context, id int) { + pluginCache := GetInstance().PluginCache + qb := j.txnManager.Gallery + + if err := txn.WithTxn(ctx, j.txnManager, func(ctx context.Context) error { + g, err := qb.Find(ctx, id) + if err != nil { + return err + } + + if g == nil { + return fmt.Errorf("gallery not found: %d", id) + } + + if err := g.LoadPrimaryFile(ctx, j.txnManager.File); err != nil { + return err + } + + if err := qb.Destroy(ctx, id); err != nil { + return err + } + + pluginCache.RegisterPostHooks(ctx, id, plugin.GalleryDestroyPost, plugin.GalleryDestroyInput{ + Checksum: g.PrimaryChecksum(), + Path: g.Path, + }, nil) + + return nil + }); err != nil { + logger.Errorf("Error deleting gallery from database: %s", err.Error()) + } +} + type cleanFilter struct { scanFilter } diff --git a/internal/manager/task_scan.go b/internal/manager/task_scan.go index 55ee9f614..5273b5636 100644 --- a/internal/manager/task_scan.go +++ b/internal/manager/task_scan.go @@ -9,6 +9,7 @@ import ( "regexp" "time" + "github.com/99designs/gqlgen/graphql/handler/lru" "github.com/stashapp/stash/internal/manager/config" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/file/video" @@ -57,11 +58,13 @@ func (j *ScanJob) Execute(ctx context.Context, progress *job.Progress) { } j.scanner.Scan(ctx, getScanHandlers(j.input, taskQueue, progress), file.ScanOptions{ - Paths: paths, - ScanFilters: []file.PathFilter{newScanFilter(instance.Config, minModTime)}, - ZipFileExtensions: instance.Config.GetGalleryExtensions(), - ParallelTasks: instance.Config.GetParallelTasksWithAutoDetection(), - HandlerRequiredFilters: []file.Filter{newHandlerRequiredFilter(instance.Config)}, + Paths: paths, + ScanFilters: []file.PathFilter{newScanFilter(instance.Config, minModTime)}, + ZipFileExtensions: instance.Config.GetGalleryExtensions(), + ParallelTasks: instance.Config.GetParallelTasksWithAutoDetection(), + HandlerRequiredFilters: []file.Filter{ + newHandlerRequiredFilter(instance.Config), + }, }, progress) taskQueue.Close() @@ -95,22 +98,31 @@ type fileCounter interface { CountByFileID(ctx context.Context, fileID file.ID) (int, error) } +type galleryFinder interface { + fileCounter + FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Gallery, error) +} + // handlerRequiredFilter returns true if a File's handler needs to be executed despite the file not being updated. type handlerRequiredFilter struct { extensionConfig SceneFinder fileCounter ImageFinder fileCounter - GalleryFinder fileCounter + GalleryFinder galleryFinder + + FolderCache *lru.LRU } func newHandlerRequiredFilter(c *config.Instance) *handlerRequiredFilter { db := instance.Database + processes := c.GetParallelTasksWithAutoDetection() return &handlerRequiredFilter{ extensionConfig: newExtensionConfig(c), SceneFinder: db.Scene, ImageFinder: db.Image, GalleryFinder: db.Gallery, + FolderCache: lru.New(processes * 2), } } @@ -143,7 +155,32 @@ func (f *handlerRequiredFilter) Accept(ctx context.Context, ff file.File) bool { } // execute handler if there are no related objects - return n == 0 + if n == 0 { + return true + } + + // if create galleries from folder is enabled and the file is not in a zip + // file, then check if there is a folder-based gallery for the file's + // directory + if isImageFile && instance.Config.GetCreateGalleriesFromFolders() && ff.Base().ZipFileID == nil { + // only do this for the first time it encounters the folder + // the first instance should create the gallery + _, found := f.FolderCache.Get(ctx, ff.Base().ParentFolderID.String()) + if found { + // should already be handled + return false + } + + g, _ := f.GalleryFinder.FindByFolderID(ctx, ff.Base().ParentFolderID) + f.FolderCache.Add(ctx, ff.Base().ParentFolderID.String(), true) + + if len(g) == 0 { + // no folder gallery. Return true so that it creates one. + return true + } + } + + return false } type scanFilter struct { @@ -248,6 +285,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre isGenerateThumbnails: options.ScanGenerateThumbnails, }, PluginCache: pluginCache, + Paths: instance.Paths, }, }, &file.FilteredHandler{ @@ -255,6 +293,7 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre Handler: &gallery.ScanHandler{ CreatorUpdater: db.Gallery, SceneFinderUpdater: db.Scene, + ImageFinderUpdater: db.Image, PluginCache: pluginCache, }, }, @@ -269,6 +308,8 @@ func getScanHandlers(options ScanMetadataInput, taskQueue *job.TaskQueue, progre taskQueue: taskQueue, progress: progress, }, + FileNamingAlgorithm: instance.Config.GetVideoFileNamingAlgorithm(), + Paths: instance.Paths, }, }, } diff --git a/pkg/file/clean.go b/pkg/file/clean.go index 05546fec3..7a1ff912a 100644 --- a/pkg/file/clean.go +++ b/pkg/file/clean.go @@ -338,7 +338,9 @@ func (j *cleanJob) shouldCleanFolder(ctx context.Context, f *Folder) bool { path := f.Path info, err := f.Info(j.FS) - if err != nil && !errors.Is(err, fs.ErrNotExist) { + // ErrInvalid can occur in zip files where the zip file path changed + // and the underlying folder did not + if err != nil && !errors.Is(err, fs.ErrNotExist) && !errors.Is(err, fs.ErrInvalid) { logger.Errorf("error getting folder info for %q, not cleaning: %v", path, err) return false } diff --git a/pkg/file/handler.go b/pkg/file/handler.go index b51b2a76a..5932968b6 100644 --- a/pkg/file/handler.go +++ b/pkg/file/handler.go @@ -29,7 +29,7 @@ func (ff FilterFunc) Accept(ctx context.Context, f File) bool { // Handler provides a handler for Files. type Handler interface { - Handle(ctx context.Context, f File) error + Handle(ctx context.Context, f File, oldFile File) error } // FilteredHandler is a Handler runs only if the filter accepts the file. @@ -39,9 +39,9 @@ type FilteredHandler struct { } // Handle runs the handler if the filter accepts the file. -func (h *FilteredHandler) Handle(ctx context.Context, f File) error { +func (h *FilteredHandler) Handle(ctx context.Context, f File, oldFile File) error { if h.Accept(ctx, f) { - return h.Handler.Handle(ctx, f) + return h.Handler.Handle(ctx, f, oldFile) } return nil } diff --git a/pkg/file/scan.go b/pkg/file/scan.go index 84c677f9a..a631b9ca7 100644 --- a/pkg/file/scan.go +++ b/pkg/file/scan.go @@ -638,7 +638,7 @@ func (s *scanJob) onNewFile(ctx context.Context, f scanFile) (File, error) { return fmt.Errorf("creating file %q: %w", path, err) } - if err := s.fireHandlers(ctx, file); err != nil { + if err := s.fireHandlers(ctx, file, nil); err != nil { return err } @@ -662,9 +662,9 @@ func (s *scanJob) fireDecorators(ctx context.Context, fs FS, f File) (File, erro return f, nil } -func (s *scanJob) fireHandlers(ctx context.Context, f File) error { +func (s *scanJob) fireHandlers(ctx context.Context, f File, oldFile File) error { for _, h := range s.handlers { - if err := h.Handle(ctx, f); err != nil { + if err := h.Handle(ctx, f, oldFile); err != nil { return err } } @@ -774,6 +774,10 @@ func (s *scanJob) handleRename(ctx context.Context, f File, fp []Fingerprint) (F return fmt.Errorf("updating file for rename %q: %w", fBase.Path, err) } + if err := s.fireHandlers(ctx, f, other); err != nil { + return err + } + return nil }); err != nil { return nil, err @@ -888,6 +892,8 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File) return s.onUnchangedFile(ctx, f, existing) } + oldBase := *base + logger.Infof("%s has been updated: rescanning", path) base.ModTime = fileModTime base.Size = f.Size @@ -914,7 +920,7 @@ func (s *scanJob) onExistingFile(ctx context.Context, f scanFile, existing File) return fmt.Errorf("updating file %q: %w", path, err) } - if err := s.fireHandlers(ctx, existing); err != nil { + if err := s.fireHandlers(ctx, existing, &oldBase); err != nil { return err } @@ -991,7 +997,7 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File } if err := s.withTxn(ctx, func(ctx context.Context) error { - if err := s.fireHandlers(ctx, existing); err != nil { + if err := s.fireHandlers(ctx, existing, nil); err != nil { return err } @@ -1002,9 +1008,5 @@ func (s *scanJob) onUnchangedFile(ctx context.Context, f scanFile, existing File // if this file is a zip file, then we need to rescan the contents // as well. We do this by returning the file, instead of nil. - if isMissingMetdata { - return existing, nil - } - - return nil, nil + return existing, nil } diff --git a/pkg/fsutil/dir.go b/pkg/fsutil/dir.go index 2bd796ed2..37f0a6d3f 100644 --- a/pkg/fsutil/dir.go +++ b/pkg/fsutil/dir.go @@ -34,6 +34,17 @@ func IsPathInDir(dir, pathToCheck string) bool { return false } +// IsPathInDirs returns true if pathToCheck is within anys of the paths in dirs. +func IsPathInDirs(dirs []string, pathToCheck string) bool { + for _, dir := range dirs { + if IsPathInDir(dir, pathToCheck) { + return true + } + } + + return false +} + // GetHomeDirectory returns the path of the user's home directory. ~ on Unix and C:\Users\UserName on Windows func GetHomeDirectory() string { currentUser, err := user.Current() diff --git a/pkg/gallery/scan.go b/pkg/gallery/scan.go index ed4ce0966..8a35890ee 100644 --- a/pkg/gallery/scan.go +++ b/pkg/gallery/scan.go @@ -27,14 +27,19 @@ type SceneFinderUpdater interface { AddGalleryIDs(ctx context.Context, sceneID int, galleryIDs []int) error } +type ImageFinderUpdater interface { + FindByZipFileID(ctx context.Context, zipFileID file.ID) ([]*models.Image, error) + UpdatePartial(ctx context.Context, id int, partial models.ImagePartial) (*models.Image, error) +} + type ScanHandler struct { CreatorUpdater FullCreatorUpdater SceneFinderUpdater SceneFinderUpdater - - PluginCache *plugin.Cache + ImageFinderUpdater ImageFinderUpdater + PluginCache *plugin.Cache } -func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { +func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error { baseFile := f.Base() // try to match the file to a gallery @@ -52,10 +57,23 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { } if len(existing) > 0 { - if err := h.associateExisting(ctx, existing, f); err != nil { + updateExisting := oldFile != nil + if err := h.associateExisting(ctx, existing, f, updateExisting); err != nil { return err } } else { + // only create galleries if there is something to put in them + // otherwise, they will be created on the fly when an image is created + images, err := h.ImageFinderUpdater.FindByZipFileID(ctx, f.Base().ID) + if err != nil { + return err + } + + if len(images) == 0 { + // don't create an empty gallery + return nil + } + // create a new gallery now := time.Now() newGallery := &models.Gallery{ @@ -71,6 +89,19 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { h.PluginCache.RegisterPostHooks(ctx, newGallery.ID, plugin.GalleryCreatePost, nil, nil) + // associate all the images in the zip file with the gallery + for _, i := range images { + if _, err := h.ImageFinderUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{ + GalleryIDs: &models.UpdateIDs{ + IDs: []int{newGallery.ID}, + Mode: models.RelationshipUpdateModeAdd, + }, + UpdatedAt: models.NewOptionalTime(now), + }); err != nil { + return fmt.Errorf("adding image %s to gallery: %w", i.Path, err) + } + } + existing = []*models.Gallery{newGallery} } @@ -81,7 +112,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { return nil } -func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File) error { +func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Gallery, f file.File, updateExisting bool) error { for _, i := range existing { if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil { return err @@ -107,6 +138,9 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. } } + if !found || updateExisting { + h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.GalleryUpdatePost, nil, nil) + } } return nil diff --git a/pkg/image/scan.go b/pkg/image/scan.go index 4caf567fc..3b96d400e 100644 --- a/pkg/image/scan.go +++ b/pkg/image/scan.go @@ -4,14 +4,17 @@ import ( "context" "errors" "fmt" + "os" "path/filepath" "time" "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" "github.com/stashapp/stash/pkg/sliceutil/intslice" + "github.com/stashapp/stash/pkg/txn" ) var ( @@ -20,6 +23,7 @@ var ( type FinderCreatorUpdater interface { FindByFileID(ctx context.Context, fileID file.ID) ([]*models.Image, error) + FindByFolderID(ctx context.Context, folderID file.FolderID) ([]*models.Image, error) FindByFingerprints(ctx context.Context, fp []file.Fingerprint) ([]*models.Image, error) Create(ctx context.Context, newImage *models.ImageCreateInput) error UpdatePartial(ctx context.Context, id int, updatedImage models.ImagePartial) (*models.Image, error) @@ -48,6 +52,8 @@ type ScanHandler struct { ScanConfig ScanConfig PluginCache *plugin.Cache + + Paths *paths.Paths } func (h *ScanHandler) validate() error { @@ -60,11 +66,34 @@ func (h *ScanHandler) validate() error { if h.ScanConfig == nil { return errors.New("ScanConfig is required") } + if h.Paths == nil { + return errors.New("Paths is required") + } return nil } -func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { +func (h *ScanHandler) logInfo(ctx context.Context, format string, args ...interface{}) { + // log at the end so that if anything fails above due to a locked database + // error and the transaction must be retried, then we shouldn't get multiple + // logs of the same thing. + txn.AddPostCompleteHook(ctx, func(ctx context.Context) error { + logger.Infof(format, args...) + return nil + }) +} + +func (h *ScanHandler) logError(ctx context.Context, format string, args ...interface{}) { + // log at the end so that if anything fails above due to a locked database + // error and the transaction must be retried, then we shouldn't get multiple + // logs of the same thing. + txn.AddPostCompleteHook(ctx, func(ctx context.Context) error { + logger.Errorf(format, args...) + return nil + }) +} + +func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error { if err := h.validate(); err != nil { return err } @@ -89,7 +118,9 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { } if len(existing) > 0 { - if err := h.associateExisting(ctx, existing, imageFile); err != nil { + updateExisting := oldFile != nil + + if err := h.associateExisting(ctx, existing, imageFile, updateExisting); err != nil { return err } } else { @@ -101,24 +132,12 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { GalleryIDs: models.NewRelatedIDs([]int{}), } - // if the file is in a zip, then associate it with the gallery - if imageFile.ZipFileID != nil { - g, err := h.GalleryFinder.FindByFileID(ctx, *imageFile.ZipFileID) - if err != nil { - return fmt.Errorf("finding gallery for zip file id %d: %w", *imageFile.ZipFileID, err) - } + h.logInfo(ctx, "%s doesn't exist. Creating new image...", f.Base().Path) - for _, gg := range g { - newImage.GalleryIDs.Add(gg.ID) - } - } else if h.ScanConfig.GetCreateGalleriesFromFolders() { - if err := h.associateFolderBasedGallery(ctx, newImage, imageFile); err != nil { - return err - } + if _, err := h.associateGallery(ctx, newImage, imageFile); err != nil { + return err } - logger.Infof("%s doesn't exist. Creating new image...", f.Base().Path) - if err := h.CreatorUpdater.Create(ctx, &models.ImageCreateInput{ Image: newImage, FileIDs: []file.ID{imageFile.ID}, @@ -131,11 +150,22 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { existing = []*models.Image{newImage} } + // remove the old thumbnail if the checksum changed - we'll regenerate it + if oldFile != nil { + oldHash := oldFile.Base().Fingerprints.GetString(file.FingerprintTypeMD5) + newHash := f.Base().Fingerprints.GetString(file.FingerprintTypeMD5) + + if oldHash != "" && newHash != "" && oldHash != newHash { + // remove cache dir of gallery + _ = os.Remove(h.Paths.Generated.GetThumbnailPath(oldHash, models.DefaultGthumbWidth)) + } + } + if h.ScanConfig.IsGenerateThumbnails() { for _, s := range existing { if err := h.ThumbnailGenerator.GenerateThumbnail(ctx, s, imageFile); err != nil { // just log if cover generation fails. We can try again on rescan - logger.Errorf("Error generating thumbnail for %s: %v", imageFile.Path, err) + h.logError(ctx, "Error generating thumbnail for %s: %v", imageFile.Path, err) } } } @@ -143,7 +173,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { return nil } -func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile) error { +func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Image, f *file.ImageFile, updateExisting bool) error { for _, i := range existing { if err := i.LoadFiles(ctx, h.CreatorUpdater); err != nil { return err @@ -157,35 +187,49 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. } } - if !found { - logger.Infof("Adding %s to image %s", f.Path, i.DisplayName()) + // associate with gallery if applicable + changed, err := h.associateGallery(ctx, i, f) + if err != nil { + return err + } - // associate with folder-based gallery if applicable - if h.ScanConfig.GetCreateGalleriesFromFolders() { - if err := h.associateFolderBasedGallery(ctx, i, f); err != nil { - return err - } + var galleryIDs *models.UpdateIDs + if changed { + galleryIDs = &models.UpdateIDs{ + IDs: i.GalleryIDs.List(), + Mode: models.RelationshipUpdateModeSet, } + } + + if !found { + h.logInfo(ctx, "Adding %s to image %s", f.Path, i.DisplayName()) if err := h.CreatorUpdater.AddFileID(ctx, i.ID, f.ID); err != nil { return fmt.Errorf("adding file to image: %w", err) } - // update updated_at time - if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.NewImagePartial()); err != nil { + + changed = true + } + + if changed { + // always update updated_at time + if _, err := h.CreatorUpdater.UpdatePartial(ctx, i.ID, models.ImagePartial{ + GalleryIDs: galleryIDs, + UpdatedAt: models.NewOptionalTime(time.Now()), + }); err != nil { return fmt.Errorf("updating image: %w", err) } } + + if changed || updateExisting { + h.PluginCache.RegisterPostHooks(ctx, i.ID, plugin.ImageUpdatePost, nil, nil) + } } return nil } func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file.File) (*models.Gallery, error) { - // don't create folder-based galleries for files in zip file - if f.Base().ZipFileID != nil { - return nil, nil - } - folderID := f.Base().ParentFolderID g, err := h.GalleryFinder.FindByFolderID(ctx, folderID) if err != nil { @@ -205,28 +249,100 @@ func (h *ScanHandler) getOrCreateFolderBasedGallery(ctx context.Context, f file. UpdatedAt: now, } - logger.Infof("Creating folder-based gallery for %s", filepath.Dir(f.Base().Path)) + h.logInfo(ctx, "Creating folder-based gallery for %s", filepath.Dir(f.Base().Path)) + if err := h.GalleryFinder.Create(ctx, newGallery, nil); err != nil { return nil, fmt.Errorf("creating folder based gallery: %w", err) } + // it's possible that there are other images in the folder that + // need to be added to the new gallery. Find and add them now. + if err := h.associateFolderImages(ctx, newGallery); err != nil { + return nil, fmt.Errorf("associating existing folder images: %w", err) + } + return newGallery, nil } -func (h *ScanHandler) associateFolderBasedGallery(ctx context.Context, newImage *models.Image, f file.File) error { - g, err := h.getOrCreateFolderBasedGallery(ctx, f) +func (h *ScanHandler) associateFolderImages(ctx context.Context, g *models.Gallery) error { + i, err := h.CreatorUpdater.FindByFolderID(ctx, *g.FolderID) if err != nil { - return err + return fmt.Errorf("finding images in folder: %w", err) } - if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil { - return err - } + for _, ii := range i { + h.logInfo(ctx, "Adding %s to gallery %s", ii.Path, g.Path) - if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) { - newImage.GalleryIDs.Add(g.ID) - logger.Infof("Adding %s to folder-based gallery %s", f.Base().Path, g.Path) + if _, err := h.CreatorUpdater.UpdatePartial(ctx, ii.ID, models.ImagePartial{ + GalleryIDs: &models.UpdateIDs{ + IDs: []int{g.ID}, + Mode: models.RelationshipUpdateModeAdd, + }, + UpdatedAt: models.NewOptionalTime(time.Now()), + }); err != nil { + return fmt.Errorf("updating image: %w", err) + } } return nil } + +func (h *ScanHandler) getOrCreateZipBasedGallery(ctx context.Context, zipFile file.File) (*models.Gallery, error) { + g, err := h.GalleryFinder.FindByFileID(ctx, zipFile.Base().ID) + if err != nil { + return nil, fmt.Errorf("finding zip based gallery: %w", err) + } + + if len(g) > 0 { + gg := g[0] + return gg, nil + } + + // create a new zip-based gallery + now := time.Now() + newGallery := &models.Gallery{ + CreatedAt: now, + UpdatedAt: now, + } + + h.logInfo(ctx, "%s doesn't exist. Creating new gallery...", zipFile.Base().Path) + + if err := h.GalleryFinder.Create(ctx, newGallery, []file.ID{zipFile.Base().ID}); err != nil { + return nil, fmt.Errorf("creating zip-based gallery: %w", err) + } + + return newGallery, nil +} + +func (h *ScanHandler) getOrCreateGallery(ctx context.Context, f file.File) (*models.Gallery, error) { + // don't create folder-based galleries for files in zip file + if f.Base().ZipFile != nil { + return h.getOrCreateZipBasedGallery(ctx, f.Base().ZipFile) + } + + if h.ScanConfig.GetCreateGalleriesFromFolders() { + return h.getOrCreateFolderBasedGallery(ctx, f) + } + + return nil, nil +} + +func (h *ScanHandler) associateGallery(ctx context.Context, newImage *models.Image, f file.File) (bool, error) { + g, err := h.getOrCreateGallery(ctx, f) + if err != nil { + return false, err + } + + if err := newImage.LoadGalleryIDs(ctx, h.CreatorUpdater); err != nil { + return false, err + } + + ret := false + if g != nil && !intslice.IntInclude(newImage.GalleryIDs.List(), g.ID) { + ret = true + newImage.GalleryIDs.Add(g.ID) + h.logInfo(ctx, "Adding %s to gallery %s", f.Base().Path, g.Path) + } + + return ret, nil +} diff --git a/pkg/models/model_file.go b/pkg/models/model_file.go index 2a1d31900..4e8ddbef8 100644 --- a/pkg/models/model_file.go +++ b/pkg/models/model_file.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "strconv" - "time" ) type HashAlgorithm string @@ -48,28 +47,3 @@ func (e *HashAlgorithm) UnmarshalGQL(v interface{}) error { func (e HashAlgorithm) MarshalGQL(w io.Writer) { fmt.Fprint(w, strconv.Quote(e.String())) } - -type File struct { - Checksum string `db:"checksum" json:"checksum"` - OSHash string `db:"oshash" json:"oshash"` - Path string `db:"path" json:"path"` - Size string `db:"size" json:"size"` - FileModTime time.Time `db:"file_mod_time" json:"file_mod_time"` -} - -// GetHash returns the hash of the scene, based on the hash algorithm provided. If -// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned. -func (s File) GetHash(hashAlgorithm HashAlgorithm) string { - switch hashAlgorithm { - case HashAlgorithmMd5: - return s.Checksum - case HashAlgorithmOshash: - return s.OSHash - default: - panic("unknown hash algorithm") - } -} - -func (s File) Equal(o File) bool { - return s.Path == o.Path && s.Checksum == o.Checksum && s.OSHash == o.OSHash && s.Size == o.Size && s.FileModTime.Equal(o.FileModTime) -} diff --git a/pkg/scene/hash.go b/pkg/scene/hash.go new file mode 100644 index 000000000..4b06a73ef --- /dev/null +++ b/pkg/scene/hash.go @@ -0,0 +1,19 @@ +package scene + +import ( + "github.com/stashapp/stash/pkg/file" + "github.com/stashapp/stash/pkg/models" +) + +// GetHash returns the hash of the file, based on the hash algorithm provided. If +// hash algorithm is MD5, then Checksum is returned. Otherwise, OSHash is returned. +func GetHash(f file.File, hashAlgorithm models.HashAlgorithm) string { + switch hashAlgorithm { + case models.HashAlgorithmMd5: + return f.Base().Fingerprints.GetString(file.FingerprintTypeMD5) + case models.HashAlgorithmOshash: + return f.Base().Fingerprints.GetString(file.FingerprintTypeOshash) + default: + panic("unknown hash algorithm") + } +} diff --git a/pkg/scene/scan.go b/pkg/scene/scan.go index f6fe8adcb..9404a0b88 100644 --- a/pkg/scene/scan.go +++ b/pkg/scene/scan.go @@ -9,6 +9,7 @@ import ( "github.com/stashapp/stash/pkg/file" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/models" + "github.com/stashapp/stash/pkg/models/paths" "github.com/stashapp/stash/pkg/plugin" ) @@ -35,6 +36,9 @@ type ScanHandler struct { CoverGenerator CoverGenerator ScanGenerator ScanGenerator PluginCache *plugin.Cache + + FileNamingAlgorithm models.HashAlgorithm + Paths *paths.Paths } func (h *ScanHandler) validate() error { @@ -47,11 +51,17 @@ func (h *ScanHandler) validate() error { if h.ScanGenerator == nil { return errors.New("ScanGenerator is required") } + if !h.FileNamingAlgorithm.IsValid() { + return errors.New("FileNamingAlgorithm is required") + } + if h.Paths == nil { + return errors.New("Paths is required") + } return nil } -func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { +func (h *ScanHandler) Handle(ctx context.Context, f file.File, oldFile file.File) error { if err := h.validate(); err != nil { return err } @@ -76,7 +86,8 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { } if len(existing) > 0 { - if err := h.associateExisting(ctx, existing, videoFile); err != nil { + updateExisting := oldFile != nil + if err := h.associateExisting(ctx, existing, videoFile, updateExisting); err != nil { return err } } else { @@ -98,6 +109,16 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { existing = []*models.Scene{newScene} } + if oldFile != nil { + // migrate hashes from the old file to the new + oldHash := GetHash(oldFile, h.FileNamingAlgorithm) + newHash := GetHash(f, h.FileNamingAlgorithm) + + if oldHash != "" && newHash != "" && oldHash != newHash { + MigrateHash(h.Paths, oldHash, newHash) + } + } + for _, s := range existing { if err := h.CoverGenerator.GenerateCover(ctx, s, videoFile); err != nil { // just log if cover generation fails. We can try again on rescan @@ -113,7 +134,7 @@ func (h *ScanHandler) Handle(ctx context.Context, f file.File) error { return nil } -func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile) error { +func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models.Scene, f *file.VideoFile, updateExisting bool) error { for _, s := range existing { if err := s.LoadFiles(ctx, h.CreatorUpdater); err != nil { return err @@ -139,6 +160,10 @@ func (h *ScanHandler) associateExisting(ctx context.Context, existing []*models. return fmt.Errorf("updating scene: %w", err) } } + + if !found || updateExisting { + h.PluginCache.RegisterPostHooks(ctx, s.ID, plugin.SceneUpdatePost, nil, nil) + } } return nil diff --git a/pkg/txn/hooks.go b/pkg/txn/hooks.go index 8ace7c3d5..13cc85d05 100644 --- a/pkg/txn/hooks.go +++ b/pkg/txn/hooks.go @@ -13,6 +13,7 @@ const ( type hookManager struct { postCommitHooks []TxnFunc postRollbackHooks []TxnFunc + postCompleteHooks []TxnFunc } func (m *hookManager) register(ctx context.Context) context.Context { @@ -27,20 +28,26 @@ func hookManagerCtx(ctx context.Context) *hookManager { return m } -func executePostCommitHooks(ctx context.Context) { - m := hookManagerCtx(ctx) - for _, h := range m.postCommitHooks { +func executeHooks(ctx context.Context, hooks []TxnFunc) { + for _, h := range hooks { // ignore errors _ = h(ctx) } } +func executePostCommitHooks(ctx context.Context) { + m := hookManagerCtx(ctx) + executeHooks(ctx, m.postCommitHooks) +} + func executePostRollbackHooks(ctx context.Context) { m := hookManagerCtx(ctx) - for _, h := range m.postRollbackHooks { - // ignore errors - _ = h(ctx) - } + executeHooks(ctx, m.postRollbackHooks) +} + +func executePostCompleteHooks(ctx context.Context) { + m := hookManagerCtx(ctx) + executeHooks(ctx, m.postCompleteHooks) } func AddPostCommitHook(ctx context.Context, hook TxnFunc) { @@ -52,3 +59,8 @@ func AddPostRollbackHook(ctx context.Context, hook TxnFunc) { m := hookManagerCtx(ctx) m.postRollbackHooks = append(m.postRollbackHooks, hook) } + +func AddPostCompleteHook(ctx context.Context, hook TxnFunc) { + m := hookManagerCtx(ctx) + m.postCompleteHooks = append(m.postCompleteHooks, hook) +} diff --git a/pkg/txn/transaction.go b/pkg/txn/transaction.go index 401286a47..0a0390382 100644 --- a/pkg/txn/transaction.go +++ b/pkg/txn/transaction.go @@ -22,6 +22,11 @@ type TxnFunc func(ctx context.Context) error // WithTxn executes fn in a transaction. If fn returns an error then // the transaction is rolled back. Otherwise it is committed. func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error { + const execComplete = true + return withTxn(ctx, m, fn, execComplete) +} + +func withTxn(ctx context.Context, m Manager, fn TxnFunc, execCompleteOnLocked bool) error { var err error ctx, err = begin(ctx, m) if err != nil { @@ -38,10 +43,16 @@ func WithTxn(ctx context.Context, m Manager, fn TxnFunc) error { if err != nil { // something went wrong, rollback rollback(ctx, m) + + if execCompleteOnLocked || !m.IsLocked(err) { + executePostCompleteHooks(ctx) + } } else { // all good, commit err = commit(ctx, m) + executePostCompleteHooks(ctx) } + }() err = fn(ctx) @@ -102,7 +113,8 @@ func (r Retryer) WithTxn(ctx context.Context, fn TxnFunc) error { var attempt int var err error for attempt = 1; attempt <= r.Retries || r.Retries < 0; attempt++ { - err = WithTxn(ctx, r.Manager, fn) + const execComplete = false + err = withTxn(ctx, r.Manager, fn, execComplete) if err == nil { return nil