Scan refactor (#1816)

* Add file scanner
* Scan scene changes
* Split scan files
* Generalise scan
* Refactor ffprobe
* Refactor ffmpeg encoder
* Move scene scan code to scene package
* Move matchExtension to utils
* Refactor gallery scanning
* Refactor image scanning
* Prevent race conditions on identical hashes
* Refactor image thumbnail generation
* Perform count concurrently
* Allow progress increment before total set
* Make progress updates more frequent

pkg/image/scan.go

@@ -0,0 +1,192 @@
+package image
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/logger"
+	"github.com/stashapp/stash/pkg/manager/paths"
+	"github.com/stashapp/stash/pkg/models"
+	"github.com/stashapp/stash/pkg/plugin"
+	"github.com/stashapp/stash/pkg/utils"
+)
+
+const mutexType = "image"
+
+type Scanner struct {
+	file.Scanner
+
+	StripFileExtension bool
+
+	Ctx             context.Context
+	CaseSensitiveFs bool
+	TxnManager      models.TransactionManager
+	Paths           *paths.Paths
+	PluginCache     *plugin.Cache
+	MutexManager    *utils.MutexManager
+}
+
+func FileScanner(hasher file.Hasher) file.Scanner {
+	return file.Scanner{
+		Hasher:       hasher,
+		CalculateMD5: true,
+	}
+}
+
+func (scanner *Scanner) ScanExisting(existing file.FileBased, file file.SourceFile) (retImage *models.Image, err error) {
+	scanned, err := scanner.Scanner.ScanExisting(existing, file)
+	if err != nil {
+		return nil, err
+	}
+
+	i := existing.(*models.Image)
+
+	path := scanned.New.Path
+	oldChecksum := i.Checksum
+	changed := false
+
+	if scanned.ContentsChanged() {
+		logger.Infof("%s has been updated: rescanning", path)
+
+		// regenerate the file details as well
+		if err := SetFileDetails(i); err != nil {
+			return nil, err
+		}
+
+		changed = true
+	} else if scanned.FileUpdated() {
+		logger.Infof("Updated image file %s", path)
+
+		changed = true
+	}
+
+	if changed {
+		i.SetFile(*scanned.New)
+		i.UpdatedAt = models.SQLiteTimestamp{Timestamp: time.Now()}
+
+		// we are operating on a checksum now, so grab a mutex on the checksum
+		done := make(chan struct{})
+		scanner.MutexManager.Claim(mutexType, scanned.New.Checksum, done)
+
+		if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error {
+			// free the mutex once transaction is complete
+			defer close(done)
+			var err error
+
+			// ensure no clashes of hashes
+			if scanned.New.Checksum != "" && scanned.Old.Checksum != scanned.New.Checksum {
+				dupe, _ := r.Image().FindByChecksum(i.Checksum)
+				if dupe != nil {
+					return fmt.Errorf("MD5 for file %s is the same as that of %s", path, dupe.Path)
+				}
+			}
+
+			retImage, err = r.Image().UpdateFull(*i)
+			return err
+		}); err != nil {
+			return nil, err
+		}
+
+		// remove the old thumbnail if the checksum changed - we'll regenerate it
+		if oldChecksum != scanned.New.Checksum {
+			// remove cache dir of gallery
+			err = os.Remove(scanner.Paths.Generated.GetThumbnailPath(oldChecksum, models.DefaultGthumbWidth))
+			if err != nil {
+				logger.Errorf("Error deleting thumbnail image: %s", err)
+			}
+		}
+
+		scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retImage.ID, plugin.ImageUpdatePost, nil, nil)
+	}
+
+	return
+}
+
+func (scanner *Scanner) ScanNew(f file.SourceFile) (retImage *models.Image, err error) {
+	scanned, err := scanner.Scanner.ScanNew(f)
+	if err != nil {
+		return nil, err
+	}
+
+	path := f.Path()
+	checksum := scanned.Checksum
+
+	// grab a mutex on the checksum
+	done := make(chan struct{})
+	scanner.MutexManager.Claim(mutexType, checksum, done)
+	defer close(done)
+
+	// check for image by checksum
+	var existingImage *models.Image
+	if err := scanner.TxnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error {
+		var err error
+		existingImage, err = r.Image().FindByChecksum(checksum)
+		return err
+	}); err != nil {
+		return nil, err
+	}
+
+	pathDisplayName := file.ZipPathDisplayName(path)
+
+	if existingImage != nil {
+		exists := FileExists(existingImage.Path)
+		if !scanner.CaseSensitiveFs {
+			// #1426 - if file exists but is a case-insensitive match for the
+			// original filename, then treat it as a move
+			if exists && strings.EqualFold(path, existingImage.Path) {
+				exists = false
+			}
+		}
+
+		if exists {
+			logger.Infof("%s already exists. Duplicate of %s ", pathDisplayName, file.ZipPathDisplayName(existingImage.Path))
+			return nil, nil
+		} else {
+			logger.Infof("%s already exists. Updating path...", pathDisplayName)
+			imagePartial := models.ImagePartial{
+				ID:   existingImage.ID,
+				Path: &path,
+			}
+
+			if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error {
+				retImage, err = r.Image().Update(imagePartial)
+				return err
+			}); err != nil {
+				return nil, err
+			}
+
+			scanner.PluginCache.ExecutePostHooks(scanner.Ctx, existingImage.ID, plugin.ImageUpdatePost, nil, nil)
+		}
+	} else {
+		logger.Infof("%s doesn't exist. Creating new item...", pathDisplayName)
+		currentTime := time.Now()
+		newImage := models.Image{
+			CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+			UpdatedAt: models.SQLiteTimestamp{Timestamp: currentTime},
+		}
+		newImage.SetFile(*scanned)
+		newImage.Title.String = GetFilename(&newImage, scanner.StripFileExtension)
+		newImage.Title.Valid = true
+
+		if err := SetFileDetails(&newImage); err != nil {
+			logger.Error(err.Error())
+			return nil, err
+		}
+
+		if err := scanner.TxnManager.WithTxn(context.TODO(), func(r models.Repository) error {
+			var err error
+			retImage, err = r.Image().Create(newImage)
+			return err
+		}); err != nil {
+			return nil, err
+		}
+
+		scanner.PluginCache.ExecutePostHooks(scanner.Ctx, retImage.ID, plugin.ImageCreatePost, nil, nil)
+	}
+
+	return
+}