Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Parallel scanning/generation, and combined scanning/preview/sprite (#820)

Browse Source 
* Implement parallel scanning and generation, and combined scanning/preview/sprite generation.
* Added UI component for preview/sprite generation during scan, and configurable number of parallel tasks.
* Add v050 changelog entry
This commit is contained in:
JoeSmithStarkers
2020-11-25 12:45:10 +11:00
committed by GitHub
parent 502e9297ad
commit e3eb550a7d
25 changed files with 445 additions and 73 deletions
Download Patch File Download Diff File Expand all files Collapse all files

108
pkg/manager/task_scan.go
View File

@@ -9,10 +9,10 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/jmoiron/sqlx"
"github.com/remeh/sizedwaitgroup"
"github.com/stashapp/stash/pkg/database"
"github.com/stashapp/stash/pkg/ffmpeg"
@@ -24,19 +24,61 @@ import (
)
type ScanTask struct {
FilePath string
UseFileMetadata bool
calculateMD5 bool
fileNamingAlgorithm models.HashAlgorithm
zipGallery *models.Gallery
FilePath string
UseFileMetadata bool
calculateMD5 bool
fileNamingAlgorithm models.HashAlgorithm
GenerateSprite bool
GeneratePreview bool
GenerateImagePreview bool
zipGallery *models.Gallery
}
func (t *ScanTask) Start(wg *sync.WaitGroup) {
func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) {
if isGallery(t.FilePath) {
t.scanGallery()
} else if isVideo(t.FilePath) {
t.scanScene()
scene := t.scanScene()
if scene != nil {
iwg := sizedwaitgroup.New(2)
if t.GenerateSprite {
iwg.Add()
taskSprite := GenerateSpriteTask{Scene: *scene, Overwrite: false, fileNamingAlgorithm: t.fileNamingAlgorithm}
go taskSprite.Start(&iwg)
}
if t.GeneratePreview {
iwg.Add()
var previewSegmentDuration = config.GetPreviewSegmentDuration()
var previewSegments = config.GetPreviewSegments()
var previewExcludeStart = config.GetPreviewExcludeStart()
var previewExcludeEnd = config.GetPreviewExcludeEnd()
var previewPresent = config.GetPreviewPreset()
// NOTE: the reuse of this model like this is painful.
previewOptions := models.GeneratePreviewOptionsInput{
PreviewSegments: &previewSegments,
PreviewSegmentDuration: &previewSegmentDuration,
PreviewExcludeStart: &previewExcludeStart,
PreviewExcludeEnd: &previewExcludeEnd,
PreviewPreset: &previewPresent,
}
taskPreview := GeneratePreviewTask{
Scene: *scene,
ImagePreview: t.GenerateImagePreview,
Options: previewOptions,
Overwrite: false,
fileNamingAlgorithm: t.fileNamingAlgorithm,
}
go taskPreview.Start(&iwg)
}
iwg.Wait()
}
} else if isImage(t.FilePath) {
t.scanImage()
}
@@ -237,7 +279,7 @@ func (t *ScanTask) isFileModified(fileModTime time.Time, modTime models.NullSQLi
}
// associates a gallery to a scene with the same basename
func (t *ScanTask) associateGallery(wg *sync.WaitGroup) {
func (t *ScanTask) associateGallery(wg *sizedwaitgroup.SizedWaitGroup) {
qb := models.NewGalleryQueryBuilder()
gallery, _ := qb.FindByPath(t.FilePath)
if gallery == nil {
@@ -291,14 +333,14 @@ func (t *ScanTask) associateGallery(wg *sync.WaitGroup) {
wg.Done()
}
func (t *ScanTask) scanScene() {
func (t *ScanTask) scanScene() *models.Scene {
qb := models.NewSceneQueryBuilder()
scene, _ := qb.FindByPath(t.FilePath)
fileModTime, err := t.getFileModTime()
if err != nil {
logger.Error(err.Error())
return
return nil
}
if scene != nil {
@@ -311,7 +353,7 @@ func (t *ScanTask) scanScene() {
scene, err = qb.Find(scene.ID)
if err != nil {
logger.Error(err.Error())
return
return nil
}
}
@@ -322,7 +364,7 @@ func (t *ScanTask) scanScene() {
scene, err = t.rescanScene(scene, fileModTime)
if err != nil {
logger.Error(err.Error())
return
return nil
}
}
@@ -335,7 +377,7 @@ func (t *ScanTask) scanScene() {
videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath)
if err != nil {
logger.Error(err.Error())
return
return nil
}
container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath)
logger.Infof("Adding container %s to file %s", container, t.FilePath)
@@ -357,14 +399,14 @@ func (t *ScanTask) scanScene() {
oshash, err := utils.OSHashFromFilePath(t.FilePath)
if err != nil {
logger.Error(err.Error())
return
return nil
}
// check if oshash clashes with existing scene
dupe, _ := qb.FindByOSHash(oshash)
if dupe != nil {
logger.Errorf("OSHash for file %s is the same as that of %s", t.FilePath, dupe.Path)
return
return nil
}
ctx := context.TODO()
@@ -373,7 +415,7 @@ func (t *ScanTask) scanScene() {
if err != nil {
logger.Error(err.Error())
tx.Rollback()
return
return nil
} else if err := tx.Commit(); err != nil {
logger.Error(err.Error())
}
@@ -384,14 +426,14 @@ func (t *ScanTask) scanScene() {
checksum, err := t.calculateChecksum()
if err != nil {
logger.Error(err.Error())
return
return nil
}
// check if checksum clashes with existing scene
dupe, _ := qb.FindByChecksum(checksum)
if dupe != nil {
logger.Errorf("MD5 for file %s is the same as that of %s", t.FilePath, dupe.Path)
return
return nil
}
ctx := context.TODO()
@@ -405,18 +447,18 @@ func (t *ScanTask) scanScene() {
}
}
return
return nil
}
// Ignore directories.
if isDir, _ := utils.DirExists(t.FilePath); isDir {
return
return nil
}
videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath)
if err != nil {
logger.Error(err.Error())
return
return nil
}
container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath)
@@ -431,14 +473,14 @@ func (t *ScanTask) scanScene() {
oshash, err := utils.OSHashFromFilePath(t.FilePath)
if err != nil {
logger.Error(err.Error())
return
return nil
}
if t.fileNamingAlgorithm == models.HashAlgorithmMd5 || t.calculateMD5 {
checksum, err = t.calculateChecksum()
if err != nil {
logger.Error(err.Error())
return
return nil
}
}
@@ -460,6 +502,8 @@ func (t *ScanTask) scanScene() {
t.makeScreenshots(videoFile, sceneHash)
var retScene *models.Scene
ctx := context.TODO()
tx := database.DB.MustBeginTx(ctx, nil)
if scene != nil {
@@ -503,15 +547,21 @@ func (t *ScanTask) scanScene() {
newScene.Details = sql.NullString{String: videoFile.Comment, Valid: true}
newScene.Date = models.SQLiteDate{String: videoFile.CreationTime.Format("2006-01-02")}
}
_, err = qb.Create(newScene, tx)
retScene, err = qb.Create(newScene, tx)
}
if err != nil {
logger.Error(err.Error())
_ = tx.Rollback()
return nil
} else if err := tx.Commit(); err != nil {
logger.Error(err.Error())
return nil
}
return retScene
}
func (t *ScanTask) rescanScene(scene *models.Scene, fileModTime time.Time) (*models.Scene, error) {
@@ -630,9 +680,9 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) {
subTask.zipGallery = zipGallery
// run the subtask and wait for it to complete
var wg sync.WaitGroup
wg.Add(1)
subTask.Start(&wg)
iwg := sizedwaitgroup.New(1)
iwg.Add()
subTask.Start(&iwg)
return nil
})
if err != nil {