diff --git a/go.mod b/go.mod index 08c335f7e..c75644f45 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/json-iterator/go v1.1.9 github.com/mattn/go-sqlite3 v1.13.0 github.com/natefinch/pie v0.0.0-20170715172608-9a0d72014007 + github.com/remeh/sizedwaitgroup v1.0.0 github.com/rs/cors v1.6.0 github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f github.com/sirupsen/logrus v1.4.2 diff --git a/go.sum b/go.sum index 7a51679a7..62cc20c93 100644 --- a/go.sum +++ b/go.sum @@ -614,6 +614,8 @@ github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= +github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.0.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.1.0 h1:g0fH8RicVgNl+zVZDCDfbdWxAWoAEJyI7I3TZYXFiig= diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index 663788299..aa068d4fc 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -9,6 +9,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { cachePath calculateMD5 videoFileNamingAlgorithm + parallelTasks previewSegments previewSegmentDuration previewExcludeStart diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 819977d24..08197f151 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -35,6 +35,8 @@ input ConfigGeneralInput { calculateMD5: Boolean! """Hash algorithm to use for generated file naming""" videoFileNamingAlgorithm: HashAlgorithm! + """Number of parallel tasks to start during scan/generate""" + parallelTasks: Int """Number of segments in a preview file""" previewSegments: Int """Preview segment duration, in seconds""" @@ -96,6 +98,8 @@ type ConfigGeneralResult { calculateMD5: Boolean! """Hash algorithm to use for generated file naming""" videoFileNamingAlgorithm: HashAlgorithm! + """Number of parallel tasks to start during scan/generate""" + parallelTasks: Int! """Number of segments in a preview file""" previewSegments: Int! """Preview segment duration, in seconds""" diff --git a/graphql/schema/types/metadata.graphql b/graphql/schema/types/metadata.graphql index 76d11de11..1722f44e4 100644 --- a/graphql/schema/types/metadata.graphql +++ b/graphql/schema/types/metadata.graphql @@ -32,7 +32,14 @@ input GeneratePreviewOptionsInput { input ScanMetadataInput { paths: [String!] + """Set name, date, details from metadata (if present)""" useFileMetadata: Boolean! + """Generate previews during scan""" + scanGeneratePreviews: Boolean! + """Generate image previews during scan""" + scanGenerateImagePreviews: Boolean! + """Generate sprites during scan""" + scanGenerateSprites: Boolean! } input AutoTagMetadataInput { diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index c95eb17ef..9b106dd04 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -61,6 +61,9 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co config.Set(config.CalculateMD5, input.CalculateMd5) + if input.ParallelTasks != nil { + config.Set(config.ParallelTasks, *input.ParallelTasks) + } if input.PreviewSegments != nil { config.Set(config.PreviewSegments, *input.PreviewSegments) } diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 6f1bc4741..c9682a24f 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -49,6 +49,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { CachePath: config.GetCachePath(), CalculateMd5: config.IsCalculateMD5(), VideoFileNamingAlgorithm: config.GetVideoFileNamingAlgorithm(), + ParallelTasks: config.GetParallelTasks(), PreviewSegments: config.GetPreviewSegments(), PreviewSegmentDuration: config.GetPreviewSegmentDuration(), PreviewExcludeStart: config.GetPreviewExcludeStart(), diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index 770b9da38..a993094d5 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -2,6 +2,7 @@ package config import ( "golang.org/x/crypto/bcrypt" + "runtime" "errors" "io/ioutil" @@ -56,6 +57,9 @@ const PreviewPreset = "preview_preset" const MaxTranscodeSize = "max_transcode_size" const MaxStreamingTranscodeSize = "max_streaming_transcode_size" +const ParallelTasks = "parallel_tasks" +const parallelTasksDefault = 1 + const PreviewSegmentDuration = "preview_segment_duration" const previewSegmentDurationDefault = 0.75 @@ -297,6 +301,20 @@ func GetPreviewSegmentDuration() float64 { return viper.GetFloat64(PreviewSegmentDuration) } +// GetParallelTasks returns the number of parallel tasks that should be started +// by scan or generate task. +func GetParallelTasks() int { + return viper.GetInt(ParallelTasks) +} + +func GetParallelTasksWithAutoDetection() int { + parallelTasks := viper.GetInt(ParallelTasks) + if parallelTasks <= 0 { + parallelTasks = (runtime.NumCPU() / 4) + 1 + } + return parallelTasks +} + // GetPreviewSegments returns the amount of segments in a scene preview file. func GetPreviewSegments() int { return viper.GetInt(PreviewSegments) @@ -550,6 +568,7 @@ func IsValid() bool { } func setDefaultValues() { + viper.SetDefault(ParallelTasks, parallelTasksDefault) viper.SetDefault(PreviewSegmentDuration, previewSegmentDurationDefault) viper.SetDefault(PreviewSegments, previewSegmentsDefault) viper.SetDefault(PreviewExcludeStart, previewExcludeStartDefault) diff --git a/pkg/manager/generator_preview.go b/pkg/manager/generator_preview.go index df19981ab..c777aa073 100644 --- a/pkg/manager/generator_preview.go +++ b/pkg/manager/generator_preview.go @@ -14,6 +14,7 @@ import ( type PreviewGenerator struct { Info *GeneratorInfo + VideoChecksum string VideoFilename string ImageFilename string OutputDirectory string @@ -26,7 +27,7 @@ type PreviewGenerator struct { Overwrite bool } -func NewPreviewGenerator(videoFile ffmpeg.VideoFile, videoFilename string, imageFilename string, outputDirectory string, generateVideo bool, generateImage bool, previewPreset string) (*PreviewGenerator, error) { +func NewPreviewGenerator(videoFile ffmpeg.VideoFile, videoChecksum string, videoFilename string, imageFilename string, outputDirectory string, generateVideo bool, generateImage bool, previewPreset string) (*PreviewGenerator, error) { exists, err := utils.FileExists(videoFile.Path) if !exists { return nil, err @@ -39,6 +40,7 @@ func NewPreviewGenerator(videoFile ffmpeg.VideoFile, videoFilename string, image return &PreviewGenerator{ Info: generator, + VideoChecksum: videoChecksum, VideoFilename: videoFilename, ImageFilename: imageFilename, OutputDirectory: outputDirectory, @@ -87,7 +89,7 @@ func (g *PreviewGenerator) generateConcatFile() error { w := bufio.NewWriter(f) for i := 0; i < g.Info.ChunkCount; i++ { num := fmt.Sprintf("%.3d", i) - filename := "preview" + num + ".mp4" + filename := "preview_" + g.VideoChecksum + "_" + num + ".mp4" _, _ = w.WriteString(fmt.Sprintf("file '%s'\n", filename)) } return w.Flush() @@ -105,7 +107,7 @@ func (g *PreviewGenerator) generateVideo(encoder *ffmpeg.Encoder, fallback bool) for i := 0; i < g.Info.ChunkCount; i++ { time := offset + (float64(i) * stepSize) num := fmt.Sprintf("%.3d", i) - filename := "preview" + num + ".mp4" + filename := "preview_" + g.VideoChecksum + "_" + num + ".mp4" chunkOutputPath := instance.Paths.Generated.GetTmpPath(filename) options := ffmpeg.ScenePreviewChunkOptions{ @@ -148,5 +150,5 @@ func (g *PreviewGenerator) generateImage(encoder *ffmpeg.Encoder) error { } func (g *PreviewGenerator) getConcatFilePath() string { - return instance.Paths.Generated.GetTmpPath("files.txt") + return instance.Paths.Generated.GetTmpPath(fmt.Sprintf("files_%s.txt", g.VideoChecksum)) } diff --git a/pkg/manager/generator_sprite.go b/pkg/manager/generator_sprite.go index 66e16e559..c2d642cb1 100644 --- a/pkg/manager/generator_sprite.go +++ b/pkg/manager/generator_sprite.go @@ -20,6 +20,7 @@ import ( type SpriteGenerator struct { Info *GeneratorInfo + VideoChecksum string ImageOutputPath string VTTOutputPath string Rows int @@ -28,7 +29,7 @@ type SpriteGenerator struct { Overwrite bool } -func NewSpriteGenerator(videoFile ffmpeg.VideoFile, imageOutputPath string, vttOutputPath string, rows int, cols int) (*SpriteGenerator, error) { +func NewSpriteGenerator(videoFile ffmpeg.VideoFile, videoChecksum string, imageOutputPath string, vttOutputPath string, rows int, cols int) (*SpriteGenerator, error) { exists, err := utils.FileExists(videoFile.Path) if !exists { return nil, err @@ -44,6 +45,7 @@ func NewSpriteGenerator(videoFile ffmpeg.VideoFile, imageOutputPath string, vttO return &SpriteGenerator{ Info: generator, + VideoChecksum: videoChecksum, ImageOutputPath: imageOutputPath, VTTOutputPath: vttOutputPath, Rows: rows, @@ -74,7 +76,7 @@ func (g *SpriteGenerator) generateSpriteImage(encoder *ffmpeg.Encoder) error { for i := 0; i < g.Info.ChunkCount; i++ { time := float64(i) * stepSize num := fmt.Sprintf("%.3d", i) - filename := "thumbnail" + num + ".jpg" + filename := "thumbnail_" + g.VideoChecksum + "_" + num + ".jpg" options := ffmpeg.ScreenshotOptions{ OutputPath: instance.Paths.Generated.GetTmpPath(filename), @@ -85,7 +87,7 @@ func (g *SpriteGenerator) generateSpriteImage(encoder *ffmpeg.Encoder) error { } // Combine all of the thumbnails into a sprite image - globPath := filepath.Join(instance.Paths.Generated.Tmp, "thumbnail*.jpg") + globPath := filepath.Join(instance.Paths.Generated.Tmp, fmt.Sprintf("thumbnail_%s_*.jpg", g.VideoChecksum)) imagePaths, _ := doublestar.Glob(globPath) utils.NaturalSort(imagePaths) var images []image.Image diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index 2c13e740e..e1f69ed31 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -2,11 +2,14 @@ package manager import ( "errors" + "fmt" "os" "strconv" "sync" "time" + "github.com/remeh/sizedwaitgroup" + "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" @@ -176,7 +179,11 @@ func (s *singleton) Scan(input models.ScanMetadataInput) { logger.Infof("Starting scan of %d files. %d New files found", *total, *newFiles) } - var wg sync.WaitGroup + start := time.Now() + parallelTasks := config.GetParallelTasksWithAutoDetection() + logger.Infof("Scan started with %d parallel tasks", parallelTasks) + wg := sizedwaitgroup.New(parallelTasks) + s.Status.Progress = 0 fileNamingAlgo := config.GetVideoFileNamingAlgorithm() calculateMD5 := config.IsCalculateMD5() @@ -201,10 +208,11 @@ func (s *singleton) Scan(input models.ScanMetadataInput) { galleries = append(galleries, path) } - wg.Add(1) - task := ScanTask{FilePath: path, UseFileMetadata: input.UseFileMetadata, fileNamingAlgorithm: fileNamingAlgo, calculateMD5: calculateMD5} + instance.Paths.Generated.EnsureTmpDir() + + wg.Add() + task := ScanTask{FilePath: path, UseFileMetadata: input.UseFileMetadata, fileNamingAlgorithm: fileNamingAlgo, calculateMD5: calculateMD5, GeneratePreview: input.ScanGeneratePreviews, GenerateImagePreview: input.ScanGenerateImagePreviews, GenerateSprite: input.ScanGenerateSprites} go task.Start(&wg) - wg.Wait() return nil }) @@ -224,9 +232,14 @@ func (s *singleton) Scan(input models.ScanMetadataInput) { return } - logger.Info("Finished scan") + wg.Wait() + instance.Paths.Generated.EmptyTmpDir() + + elapsed := time.Since(start) + logger.Info(fmt.Sprintf("Scan finished (%s)", elapsed)) + for _, path := range galleries { - wg.Add(1) + wg.Add() task := ScanTask{FilePath: path, UseFileMetadata: false} go task.associateGallery(&wg) wg.Wait() @@ -357,8 +370,10 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) { return } - delta := utils.Btoi(input.Sprites) + utils.Btoi(input.Previews) + utils.Btoi(input.Markers) + utils.Btoi(input.Transcodes) - var wg sync.WaitGroup + parallelTasks := config.GetParallelTasksWithAutoDetection() + + logger.Infof("Generate started with %d parallel tasks", parallelTasks) + wg := sizedwaitgroup.New(parallelTasks) s.Status.Progress = 0 lenScenes := len(scenes) @@ -397,6 +412,10 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) { } setGeneratePreviewOptionsInput(generatePreviewOptions) + // Start measuring how long the scan has taken. (consider moving this up) + start := time.Now() + instance.Paths.Generated.EnsureTmpDir() + for i, scene := range scenes { s.Status.setProgress(i, total) if s.Status.stopping { @@ -409,15 +428,9 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) { continue } - wg.Add(delta) - - // Clear the tmp directory for each scene - if input.Sprites || input.Previews || input.Markers { - instance.Paths.Generated.EmptyTmpDir() - } - if input.Sprites { task := GenerateSpriteTask{Scene: *scene, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo} + wg.Add() go task.Start(&wg) } @@ -429,22 +442,25 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) { Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo, } + wg.Add() go task.Start(&wg) } if input.Markers { + wg.Add() task := GenerateMarkersTask{Scene: scene, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo} go task.Start(&wg) } if input.Transcodes { + wg.Add() task := GenerateTranscodeTask{Scene: *scene, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo} go task.Start(&wg) } - - wg.Wait() } + wg.Wait() + for i, marker := range markers { s.Status.setProgress(lenScenes+i, total) if s.Status.stopping { @@ -457,13 +473,16 @@ func (s *singleton) Generate(input models.GenerateMetadataInput) { continue } - wg.Add(1) + wg.Add() task := GenerateMarkersTask{Marker: marker, Overwrite: overwrite, fileNamingAlgorithm: fileNamingAlgo} go task.Start(&wg) - wg.Wait() } - logger.Infof("Generate finished") + wg.Wait() + + instance.Paths.Generated.EmptyTmpDir() + elapsed := time.Since(start) + logger.Info(fmt.Sprintf("Generate finished (%s)", elapsed)) }() } diff --git a/pkg/manager/task_generate_markers.go b/pkg/manager/task_generate_markers.go index deae76ec0..798ff21d0 100644 --- a/pkg/manager/task_generate_markers.go +++ b/pkg/manager/task_generate_markers.go @@ -3,7 +3,8 @@ package manager import ( "path/filepath" "strconv" - "sync" + + "github.com/remeh/sizedwaitgroup" "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" @@ -18,7 +19,7 @@ type GenerateMarkersTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GenerateMarkersTask) Start(wg *sync.WaitGroup) { +func (t *GenerateMarkersTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { defer wg.Done() if t.Scene != nil { diff --git a/pkg/manager/task_generate_preview.go b/pkg/manager/task_generate_preview.go index d39240fbd..30bd89945 100644 --- a/pkg/manager/task_generate_preview.go +++ b/pkg/manager/task_generate_preview.go @@ -1,7 +1,7 @@ package manager import ( - "sync" + "github.com/remeh/sizedwaitgroup" "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" @@ -19,11 +19,13 @@ type GeneratePreviewTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GeneratePreviewTask) Start(wg *sync.WaitGroup) { +func (t *GeneratePreviewTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { defer wg.Done() videoFilename := t.videoFilename() + videoChecksum := t.Scene.GetHash(t.fileNamingAlgorithm) imageFilename := t.imageFilename() + if !t.Overwrite && !t.required() { return } @@ -35,7 +37,8 @@ func (t *GeneratePreviewTask) Start(wg *sync.WaitGroup) { } const generateVideo = true - generator, err := NewPreviewGenerator(*videoFile, videoFilename, imageFilename, instance.Paths.Generated.Screenshots, generateVideo, t.ImagePreview, t.Options.PreviewPreset.String()) + generator, err := NewPreviewGenerator(*videoFile, videoChecksum, videoFilename, imageFilename, instance.Paths.Generated.Screenshots, generateVideo, t.ImagePreview, t.Options.PreviewPreset.String()) + if err != nil { logger.Errorf("error creating preview generator: %s", err.Error()) return diff --git a/pkg/manager/task_generate_sprite.go b/pkg/manager/task_generate_sprite.go index b43232557..f6ffc7ab8 100644 --- a/pkg/manager/task_generate_sprite.go +++ b/pkg/manager/task_generate_sprite.go @@ -1,7 +1,7 @@ package manager import ( - "sync" + "github.com/remeh/sizedwaitgroup" "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" @@ -15,7 +15,7 @@ type GenerateSpriteTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GenerateSpriteTask) Start(wg *sync.WaitGroup) { +func (t *GenerateSpriteTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { defer wg.Done() if !t.Overwrite && !t.required() { @@ -31,7 +31,8 @@ func (t *GenerateSpriteTask) Start(wg *sync.WaitGroup) { sceneHash := t.Scene.GetHash(t.fileNamingAlgorithm) imagePath := instance.Paths.Scene.GetSpriteImageFilePath(sceneHash) vttPath := instance.Paths.Scene.GetSpriteVttFilePath(sceneHash) - generator, err := NewSpriteGenerator(*videoFile, imagePath, vttPath, 9, 9) + generator, err := NewSpriteGenerator(*videoFile, sceneHash, imagePath, vttPath, 9, 9) + if err != nil { logger.Errorf("error creating sprite generator: %s", err.Error()) return diff --git a/pkg/manager/task_scan.go b/pkg/manager/task_scan.go index c1e7899a2..aa7fe65f7 100644 --- a/pkg/manager/task_scan.go +++ b/pkg/manager/task_scan.go @@ -9,10 +9,10 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/jmoiron/sqlx" + "github.com/remeh/sizedwaitgroup" "github.com/stashapp/stash/pkg/database" "github.com/stashapp/stash/pkg/ffmpeg" @@ -24,19 +24,61 @@ import ( ) type ScanTask struct { - FilePath string - UseFileMetadata bool - calculateMD5 bool - fileNamingAlgorithm models.HashAlgorithm - - zipGallery *models.Gallery + FilePath string + UseFileMetadata bool + calculateMD5 bool + fileNamingAlgorithm models.HashAlgorithm + GenerateSprite bool + GeneratePreview bool + GenerateImagePreview bool + zipGallery *models.Gallery } -func (t *ScanTask) Start(wg *sync.WaitGroup) { +func (t *ScanTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { if isGallery(t.FilePath) { t.scanGallery() } else if isVideo(t.FilePath) { - t.scanScene() + scene := t.scanScene() + + if scene != nil { + iwg := sizedwaitgroup.New(2) + + if t.GenerateSprite { + iwg.Add() + taskSprite := GenerateSpriteTask{Scene: *scene, Overwrite: false, fileNamingAlgorithm: t.fileNamingAlgorithm} + go taskSprite.Start(&iwg) + } + + if t.GeneratePreview { + iwg.Add() + + var previewSegmentDuration = config.GetPreviewSegmentDuration() + var previewSegments = config.GetPreviewSegments() + var previewExcludeStart = config.GetPreviewExcludeStart() + var previewExcludeEnd = config.GetPreviewExcludeEnd() + var previewPresent = config.GetPreviewPreset() + + // NOTE: the reuse of this model like this is painful. + previewOptions := models.GeneratePreviewOptionsInput{ + PreviewSegments: &previewSegments, + PreviewSegmentDuration: &previewSegmentDuration, + PreviewExcludeStart: &previewExcludeStart, + PreviewExcludeEnd: &previewExcludeEnd, + PreviewPreset: &previewPresent, + } + + taskPreview := GeneratePreviewTask{ + Scene: *scene, + ImagePreview: t.GenerateImagePreview, + Options: previewOptions, + Overwrite: false, + fileNamingAlgorithm: t.fileNamingAlgorithm, + } + go taskPreview.Start(&iwg) + } + + iwg.Wait() + } } else if isImage(t.FilePath) { t.scanImage() } @@ -237,7 +279,7 @@ func (t *ScanTask) isFileModified(fileModTime time.Time, modTime models.NullSQLi } // associates a gallery to a scene with the same basename -func (t *ScanTask) associateGallery(wg *sync.WaitGroup) { +func (t *ScanTask) associateGallery(wg *sizedwaitgroup.SizedWaitGroup) { qb := models.NewGalleryQueryBuilder() gallery, _ := qb.FindByPath(t.FilePath) if gallery == nil { @@ -291,14 +333,14 @@ func (t *ScanTask) associateGallery(wg *sync.WaitGroup) { wg.Done() } -func (t *ScanTask) scanScene() { +func (t *ScanTask) scanScene() *models.Scene { qb := models.NewSceneQueryBuilder() scene, _ := qb.FindByPath(t.FilePath) fileModTime, err := t.getFileModTime() if err != nil { logger.Error(err.Error()) - return + return nil } if scene != nil { @@ -311,7 +353,7 @@ func (t *ScanTask) scanScene() { scene, err = qb.Find(scene.ID) if err != nil { logger.Error(err.Error()) - return + return nil } } @@ -322,7 +364,7 @@ func (t *ScanTask) scanScene() { scene, err = t.rescanScene(scene, fileModTime) if err != nil { logger.Error(err.Error()) - return + return nil } } @@ -335,7 +377,7 @@ func (t *ScanTask) scanScene() { videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath) if err != nil { logger.Error(err.Error()) - return + return nil } container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath) logger.Infof("Adding container %s to file %s", container, t.FilePath) @@ -357,14 +399,14 @@ func (t *ScanTask) scanScene() { oshash, err := utils.OSHashFromFilePath(t.FilePath) if err != nil { logger.Error(err.Error()) - return + return nil } // check if oshash clashes with existing scene dupe, _ := qb.FindByOSHash(oshash) if dupe != nil { logger.Errorf("OSHash for file %s is the same as that of %s", t.FilePath, dupe.Path) - return + return nil } ctx := context.TODO() @@ -373,7 +415,7 @@ func (t *ScanTask) scanScene() { if err != nil { logger.Error(err.Error()) tx.Rollback() - return + return nil } else if err := tx.Commit(); err != nil { logger.Error(err.Error()) } @@ -384,14 +426,14 @@ func (t *ScanTask) scanScene() { checksum, err := t.calculateChecksum() if err != nil { logger.Error(err.Error()) - return + return nil } // check if checksum clashes with existing scene dupe, _ := qb.FindByChecksum(checksum) if dupe != nil { logger.Errorf("MD5 for file %s is the same as that of %s", t.FilePath, dupe.Path) - return + return nil } ctx := context.TODO() @@ -405,18 +447,18 @@ func (t *ScanTask) scanScene() { } } - return + return nil } // Ignore directories. if isDir, _ := utils.DirExists(t.FilePath); isDir { - return + return nil } videoFile, err := ffmpeg.NewVideoFile(instance.FFProbePath, t.FilePath) if err != nil { logger.Error(err.Error()) - return + return nil } container := ffmpeg.MatchContainer(videoFile.Container, t.FilePath) @@ -431,14 +473,14 @@ func (t *ScanTask) scanScene() { oshash, err := utils.OSHashFromFilePath(t.FilePath) if err != nil { logger.Error(err.Error()) - return + return nil } if t.fileNamingAlgorithm == models.HashAlgorithmMd5 || t.calculateMD5 { checksum, err = t.calculateChecksum() if err != nil { logger.Error(err.Error()) - return + return nil } } @@ -460,6 +502,8 @@ func (t *ScanTask) scanScene() { t.makeScreenshots(videoFile, sceneHash) + var retScene *models.Scene + ctx := context.TODO() tx := database.DB.MustBeginTx(ctx, nil) if scene != nil { @@ -503,15 +547,21 @@ func (t *ScanTask) scanScene() { newScene.Details = sql.NullString{String: videoFile.Comment, Valid: true} newScene.Date = models.SQLiteDate{String: videoFile.CreationTime.Format("2006-01-02")} } - _, err = qb.Create(newScene, tx) + + retScene, err = qb.Create(newScene, tx) } if err != nil { logger.Error(err.Error()) _ = tx.Rollback() + return nil + } else if err := tx.Commit(); err != nil { logger.Error(err.Error()) + return nil } + + return retScene } func (t *ScanTask) rescanScene(scene *models.Scene, fileModTime time.Time) (*models.Scene, error) { @@ -630,9 +680,9 @@ func (t *ScanTask) scanZipImages(zipGallery *models.Gallery) { subTask.zipGallery = zipGallery // run the subtask and wait for it to complete - var wg sync.WaitGroup - wg.Add(1) - subTask.Start(&wg) + iwg := sizedwaitgroup.New(1) + iwg.Add() + subTask.Start(&iwg) return nil }) if err != nil { diff --git a/pkg/manager/task_transcode.go b/pkg/manager/task_transcode.go index d21ec4059..44af08c48 100644 --- a/pkg/manager/task_transcode.go +++ b/pkg/manager/task_transcode.go @@ -1,12 +1,13 @@ package manager import ( + "github.com/remeh/sizedwaitgroup" + "github.com/stashapp/stash/pkg/ffmpeg" "github.com/stashapp/stash/pkg/logger" "github.com/stashapp/stash/pkg/manager/config" "github.com/stashapp/stash/pkg/models" "github.com/stashapp/stash/pkg/utils" - "sync" ) type GenerateTranscodeTask struct { @@ -15,7 +16,7 @@ type GenerateTranscodeTask struct { fileNamingAlgorithm models.HashAlgorithm } -func (t *GenerateTranscodeTask) Start(wg *sync.WaitGroup) { +func (t *GenerateTranscodeTask) Start(wg *sizedwaitgroup.SizedWaitGroup) { defer wg.Done() hasTranscode := HasTranscode(&t.Scene, t.fileNamingAlgorithm) diff --git a/ui/v2.5/src/components/Changelog/Changelog.tsx b/ui/v2.5/src/components/Changelog/Changelog.tsx index fb955ccd1..250ba2534 100644 --- a/ui/v2.5/src/components/Changelog/Changelog.tsx +++ b/ui/v2.5/src/components/Changelog/Changelog.tsx @@ -7,6 +7,7 @@ import V020 from "./versions/v020.md"; import V021 from "./versions/v021.md"; import V030 from "./versions/v030.md"; import V040 from "./versions/v040.md"; +import V050 from "./versions/v050.md"; import { MarkdownPage } from "../Shared/MarkdownPage"; const Changelog: React.FC = () => { @@ -36,11 +37,20 @@ const Changelog: React.FC = () => { <>

Changelog:

+ + + diff --git a/ui/v2.5/src/components/Changelog/versions/v050.md b/ui/v2.5/src/components/Changelog/versions/v050.md new file mode 100644 index 000000000..2150d257f --- /dev/null +++ b/ui/v2.5/src/components/Changelog/versions/v050.md @@ -0,0 +1,3 @@ +### 🎨 Improvements +* Support optional preview and sprite generation during scanning. +* Support configurable number of threads for scanning and generation. diff --git a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx index 336d375fa..4daaff39c 100644 --- a/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsConfigurationPanel.tsx @@ -79,6 +79,7 @@ export const SettingsConfigurationPanel: React.FC = () => { const [videoFileNamingAlgorithm, setVideoFileNamingAlgorithm] = useState< GQL.HashAlgorithm | undefined >(undefined); + const [parallelTasks, setParallelTasks] = useState(0); const [previewSegments, setPreviewSegments] = useState(0); const [previewSegmentDuration, setPreviewSegmentDuration] = useState( 0 @@ -139,6 +140,7 @@ export const SettingsConfigurationPanel: React.FC = () => { calculateMD5, videoFileNamingAlgorithm: (videoFileNamingAlgorithm as GQL.HashAlgorithm) ?? undefined, + parallelTasks, previewSegments, previewSegmentDuration, previewExcludeStart, @@ -182,6 +184,7 @@ export const SettingsConfigurationPanel: React.FC = () => { setCachePath(conf.general.cachePath); setVideoFileNamingAlgorithm(conf.general.videoFileNamingAlgorithm); setCalculateMD5(conf.general.calculateMD5); + setParallelTasks(conf.general.parallelTasks); setPreviewSegments(conf.general.previewSegments); setPreviewSegmentDuration(conf.general.previewSegmentDuration); setPreviewExcludeStart(conf.general.previewExcludeStart); @@ -567,6 +570,31 @@ export const SettingsConfigurationPanel: React.FC = () => {
+ +

Parallel Scan/Generation

+ + +
Number of parallel task for scan/generation
+ ) => + setParallelTasks( + Number.parseInt(e.currentTarget.value || "0", 10) + ) + } + /> + + Set to 0 for auto-detection. Warning running more tasks than is + required to achieve 100% cpu utilisation will decrease performance + and potentially cause other issues. + + + + +
+

Preview Generation

diff --git a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx index 533309ec5..fc4ee8604 100644 --- a/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx +++ b/ui/v2.5/src/components/Settings/SettingsTasksPanel/SettingsTasksPanel.tsx @@ -31,6 +31,16 @@ export const SettingsTasksPanel: React.FC = () => { const [isImportDialogOpen, setIsImportDialogOpen] = useState(false); const [isScanDialogOpen, setIsScanDialogOpen] = useState(false); const [useFileMetadata, setUseFileMetadata] = useState(false); + const [scanGeneratePreviews, setScanGeneratePreviews] = useState( + false + ); + const [scanGenerateSprites, setScanGenerateSprites] = useState( + false + ); + const [scanGenerateImagePreviews, setScanGenerateImagePreviews] = useState< + boolean + >(false); + const [status, setStatus] = useState(""); const [progress, setProgress] = useState(0); @@ -166,8 +176,11 @@ export const SettingsTasksPanel: React.FC = () => { async function onScan(paths?: string[]) { try { await mutateMetadataScan({ - useFileMetadata, paths, + useFileMetadata, + scanGeneratePreviews, + scanGenerateImagePreviews, + scanGenerateSprites, }); Toast.success({ content: "Started scan" }); jobStatus.refetch(); @@ -304,6 +317,31 @@ export const SettingsTasksPanel: React.FC = () => { label="Set name, date, details from metadata (if present)" onChange={() => setUseFileMetadata(!useFileMetadata)} /> + setScanGeneratePreviews(!scanGeneratePreviews)} + /> +
+
↳
+ + setScanGenerateImagePreviews(!scanGenerateImagePreviews) + } + className="ml-2 flex-grow" + /> +
+ setScanGenerateSprites(!scanGenerateSprites)} + />