mirror of
https://github.com/stashapp/stash.git
synced 2025-12-18 04:44:37 +03:00
File storage rewrite (#2676)
* Restructure data layer part 2 (#2599) * Refactor and separate image model * Refactor image query builder * Handle relationships in image query builder * Remove relationship management methods * Refactor gallery model/query builder * Add scenes to gallery model * Convert scene model * Refactor scene models * Remove unused methods * Add unit tests for gallery * Add image tests * Add scene tests * Convert unnecessary scene value pointers to values * Convert unnecessary pointer values to values * Refactor scene partial * Add scene partial tests * Refactor ImagePartial * Add image partial tests * Refactor gallery partial update * Add partial gallery update tests * Use zero/null package for null values * Add files and scan system * Add sqlite implementation for files/folders * Add unit tests for files/folders * Image refactors * Update image data layer * Refactor gallery model and creation * Refactor scene model * Refactor scenes * Don't set title from filename * Allow galleries to freely add/remove images * Add multiple scene file support to graphql and UI * Add multiple file support for images in graphql/UI * Add multiple file for galleries in graphql/UI * Remove use of some deprecated fields * Remove scene path usage * Remove gallery path usage * Remove path from image * Move funscript to video file * Refactor caption detection * Migrate existing data * Add post commit/rollback hook system * Lint. Comment out import/export tests * Add WithDatabase read only wrapper * Prepend tasks to list * Add 32 pre-migration * Add warnings in release and migration notes
This commit is contained in:
WithoutPants
@@ -1,129 +1,116 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
// type sceneScreenshotter struct {
|
||||
// g *generate.Generator
|
||||
// }
|
||||
|
||||
"github.com/stashapp/stash/internal/manager/config"
|
||||
"github.com/stashapp/stash/pkg/ffmpeg"
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/scene"
|
||||
"github.com/stashapp/stash/pkg/scene/generate"
|
||||
)
|
||||
// func (ss *sceneScreenshotter) GenerateScreenshot(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error {
|
||||
// return ss.g.Screenshot(ctx, probeResult.Path, hash, probeResult.Width, probeResult.Duration, generate.ScreenshotOptions{})
|
||||
// }
|
||||
|
||||
type sceneScreenshotter struct {
|
||||
g *generate.Generator
|
||||
}
|
||||
// func (ss *sceneScreenshotter) GenerateThumbnail(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error {
|
||||
// return ss.g.Thumbnail(ctx, probeResult.Path, hash, probeResult.Duration, generate.ScreenshotOptions{})
|
||||
// }
|
||||
|
||||
func (ss *sceneScreenshotter) GenerateScreenshot(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error {
|
||||
return ss.g.Screenshot(ctx, probeResult.Path, hash, probeResult.Width, probeResult.Duration, generate.ScreenshotOptions{})
|
||||
}
|
||||
// func (t *ScanTask) scanScene(ctx context.Context) *models.Scene {
|
||||
// logError := func(err error) *models.Scene {
|
||||
// logger.Error(err.Error())
|
||||
// return nil
|
||||
// }
|
||||
|
||||
func (ss *sceneScreenshotter) GenerateThumbnail(ctx context.Context, probeResult *ffmpeg.VideoFile, hash string) error {
|
||||
return ss.g.Thumbnail(ctx, probeResult.Path, hash, probeResult.Duration, generate.ScreenshotOptions{})
|
||||
}
|
||||
// var retScene *models.Scene
|
||||
// var s *models.Scene
|
||||
|
||||
func (t *ScanTask) scanScene(ctx context.Context) *models.Scene {
|
||||
logError := func(err error) *models.Scene {
|
||||
logger.Error(err.Error())
|
||||
return nil
|
||||
}
|
||||
// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
// var err error
|
||||
// s, err = t.TxnManager.Scene.FindByPath(ctx, t.file.Path())
|
||||
// return err
|
||||
// }); err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// return nil
|
||||
// }
|
||||
|
||||
var retScene *models.Scene
|
||||
var s *models.Scene
|
||||
// g := &generate.Generator{
|
||||
// Encoder: instance.FFMPEG,
|
||||
// LockManager: instance.ReadLockManager,
|
||||
// ScenePaths: instance.Paths.Scene,
|
||||
// }
|
||||
|
||||
if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
s, err = t.TxnManager.Scene.FindByPath(ctx, t.file.Path())
|
||||
return err
|
||||
}); err != nil {
|
||||
logger.Error(err.Error())
|
||||
return nil
|
||||
}
|
||||
// scanner := scene.Scanner{
|
||||
// Scanner: scene.FileScanner(&file.FSHasher{}, t.fileNamingAlgorithm, t.calculateMD5),
|
||||
// StripFileExtension: t.StripFileExtension,
|
||||
// FileNamingAlgorithm: t.fileNamingAlgorithm,
|
||||
// TxnManager: t.TxnManager,
|
||||
// CreatorUpdater: t.TxnManager.Scene,
|
||||
// Paths: GetInstance().Paths,
|
||||
// CaseSensitiveFs: t.CaseSensitiveFs,
|
||||
// Screenshotter: &sceneScreenshotter{
|
||||
// g: g,
|
||||
// },
|
||||
// VideoFileCreator: &instance.FFProbe,
|
||||
// PluginCache: instance.PluginCache,
|
||||
// MutexManager: t.mutexManager,
|
||||
// UseFileMetadata: t.UseFileMetadata,
|
||||
// }
|
||||
|
||||
g := &generate.Generator{
|
||||
Encoder: instance.FFMPEG,
|
||||
LockManager: instance.ReadLockManager,
|
||||
ScenePaths: instance.Paths.Scene,
|
||||
}
|
||||
// if s != nil {
|
||||
// if err := scanner.ScanExisting(ctx, s, t.file); err != nil {
|
||||
// return logError(err)
|
||||
// }
|
||||
|
||||
scanner := scene.Scanner{
|
||||
Scanner: scene.FileScanner(&file.FSHasher{}, t.fileNamingAlgorithm, t.calculateMD5),
|
||||
StripFileExtension: t.StripFileExtension,
|
||||
FileNamingAlgorithm: t.fileNamingAlgorithm,
|
||||
TxnManager: t.TxnManager,
|
||||
CreatorUpdater: t.TxnManager.Scene,
|
||||
Paths: GetInstance().Paths,
|
||||
CaseSensitiveFs: t.CaseSensitiveFs,
|
||||
Screenshotter: &sceneScreenshotter{
|
||||
g: g,
|
||||
},
|
||||
VideoFileCreator: &instance.FFProbe,
|
||||
PluginCache: instance.PluginCache,
|
||||
MutexManager: t.mutexManager,
|
||||
UseFileMetadata: t.UseFileMetadata,
|
||||
}
|
||||
// return nil
|
||||
// }
|
||||
|
||||
if s != nil {
|
||||
if err := scanner.ScanExisting(ctx, s, t.file); err != nil {
|
||||
return logError(err)
|
||||
}
|
||||
// var err error
|
||||
// retScene, err = scanner.ScanNew(ctx, t.file)
|
||||
// if err != nil {
|
||||
// return logError(err)
|
||||
// }
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
retScene, err = scanner.ScanNew(ctx, t.file)
|
||||
if err != nil {
|
||||
return logError(err)
|
||||
}
|
||||
|
||||
return retScene
|
||||
}
|
||||
// return retScene
|
||||
// }
|
||||
|
||||
// associates captions to scene/s with the same basename
|
||||
func (t *ScanTask) associateCaptions(ctx context.Context) {
|
||||
vExt := config.GetInstance().GetVideoExtensions()
|
||||
captionPath := t.file.Path()
|
||||
captionLang := scene.GetCaptionsLangFromPath(captionPath)
|
||||
// func (t *ScanTask) associateCaptions(ctx context.Context) {
|
||||
// vExt := config.GetInstance().GetVideoExtensions()
|
||||
// captionPath := t.file.Path()
|
||||
// captionLang := scene.GetCaptionsLangFromPath(captionPath)
|
||||
|
||||
relatedFiles := scene.GenerateCaptionCandidates(captionPath, vExt)
|
||||
if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
sqb := t.TxnManager.Scene
|
||||
// relatedFiles := scene.GenerateCaptionCandidates(captionPath, vExt)
|
||||
// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
// var err error
|
||||
// sqb := t.TxnManager.Scene
|
||||
|
||||
for _, scenePath := range relatedFiles {
|
||||
s, er := sqb.FindByPath(ctx, scenePath)
|
||||
// for _, scenePath := range relatedFiles {
|
||||
// s, er := sqb.FindByPath(ctx, scenePath)
|
||||
|
||||
if er != nil {
|
||||
logger.Errorf("Error searching for scene %s: %v", scenePath, er)
|
||||
continue
|
||||
}
|
||||
if s != nil { // found related Scene
|
||||
logger.Debugf("Matched captions to scene %s", s.Path)
|
||||
captions, er := sqb.GetCaptions(ctx, s.ID)
|
||||
if er == nil {
|
||||
fileExt := filepath.Ext(captionPath)
|
||||
ext := fileExt[1:]
|
||||
if !scene.IsLangInCaptions(captionLang, ext, captions) { // only update captions if language code is not present
|
||||
newCaption := &models.SceneCaption{
|
||||
LanguageCode: captionLang,
|
||||
Filename: filepath.Base(captionPath),
|
||||
CaptionType: ext,
|
||||
}
|
||||
captions = append(captions, newCaption)
|
||||
er = sqb.UpdateCaptions(ctx, s.ID, captions)
|
||||
if er == nil {
|
||||
logger.Debugf("Updated captions for scene %s. Added %s", s.Path, captionLang)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return err
|
||||
}); err != nil {
|
||||
logger.Error(err.Error())
|
||||
}
|
||||
}
|
||||
// if er != nil {
|
||||
// logger.Errorf("Error searching for scene %s: %v", scenePath, er)
|
||||
// continue
|
||||
// }
|
||||
// if s != nil { // found related Scene
|
||||
// logger.Debugf("Matched captions to scene %s", s.Path)
|
||||
// captions, er := sqb.GetCaptions(ctx, s.ID)
|
||||
// if er == nil {
|
||||
// fileExt := filepath.Ext(captionPath)
|
||||
// ext := fileExt[1:]
|
||||
// if !scene.IsLangInCaptions(captionLang, ext, captions) { // only update captions if language code is not present
|
||||
// newCaption := &models.SceneCaption{
|
||||
// LanguageCode: captionLang,
|
||||
// Filename: filepath.Base(captionPath),
|
||||
// CaptionType: ext,
|
||||
// }
|
||||
// captions = append(captions, newCaption)
|
||||
// er = sqb.UpdateCaptions(ctx, s.ID, captions)
|
||||
// if er == nil {
|
||||
// logger.Debugf("Updated captions for scene %s. Added %s", s.Path, captionLang)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return err
|
||||
// }); err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// }
|
||||
// }
|
||||
|
||||
Reference in New Issue
Block a user