Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

File storage rewrite (#2676)

Browse Source 
* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes
This commit is contained in:
WithoutPants
2022-07-13 16:30:54 +10:00
parent 30877c75fb
commit 5495d72849
359 changed files with 43690 additions and 16000 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
internal/manager/manager.go
View File

@@ -18,17 +18,27 @@ import (
"github.com/stashapp/stash/internal/log"
"github.com/stashapp/stash/internal/manager/config"
"github.com/stashapp/stash/pkg/ffmpeg"
"github.com/stashapp/stash/pkg/file"
file_image "github.com/stashapp/stash/pkg/file/image"
"github.com/stashapp/stash/pkg/file/video"
"github.com/stashapp/stash/pkg/fsutil"
"github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/image"
"github.com/stashapp/stash/pkg/job"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/models/paths"
"github.com/stashapp/stash/pkg/plugin"
"github.com/stashapp/stash/pkg/scene"
"github.com/stashapp/stash/pkg/scene/generate"
"github.com/stashapp/stash/pkg/scraper"
"github.com/stashapp/stash/pkg/session"
"github.com/stashapp/stash/pkg/sqlite"
"github.com/stashapp/stash/pkg/utils"
"github.com/stashapp/stash/ui"
// register custom migrations
_ "github.com/stashapp/stash/pkg/sqlite/migrations"
)
type SystemStatus struct {
@@ -115,7 +125,14 @@ type Manager struct {
DLNAService *dlna.Service
Database *sqlite.Database
Repository models.Repository
Repository Repository
SceneService SceneService
ImageService ImageService
GalleryService GalleryService
Scanner *file.Scanner
Cleaner *file.Cleaner
scanSubs *subscriptionManager
}
@@ -150,7 +167,7 @@ func initialize() error {
l := initLog()
initProfiling(cfg.GetCPUProfilePath())
db := &sqlite.Database{}
db := sqlite.NewDatabase()
instance = &Manager{
Config: cfg,
@@ -159,24 +176,29 @@ func initialize() error {
DownloadStore: NewDownloadStore(),
PluginCache: plugin.NewCache(cfg),
Database: db,
Repository: models.Repository{
TxnManager: db,
Gallery: sqlite.GalleryReaderWriter,
Image: sqlite.ImageReaderWriter,
Movie: sqlite.MovieReaderWriter,
Performer: sqlite.PerformerReaderWriter,
Scene: sqlite.SceneReaderWriter,
SceneMarker: sqlite.SceneMarkerReaderWriter,
ScrapedItem: sqlite.ScrapedItemReaderWriter,
Studio: sqlite.StudioReaderWriter,
Tag: sqlite.TagReaderWriter,
SavedFilter: sqlite.SavedFilterReaderWriter,
},
Database: db,
Repository: sqliteRepository(db),
scanSubs: &subscriptionManager{},
}
instance.SceneService = &scene.Service{
File: db.File,
Repository: db.Scene,
MarkerDestroyer: instance.Repository.SceneMarker,
}
instance.ImageService = &image.Service{
File: db.File,
Repository: db.Image,
}
instance.GalleryService = &gallery.Service{
Repository: db.Gallery,
ImageFinder: db.Image,
ImageService: instance.ImageService,
}
instance.JobManager = initJobManager()
sceneServer := SceneServer{
@@ -200,13 +222,15 @@ func initialize() error {
}
if err != nil {
panic(fmt.Sprintf("error initializing configuration: %s", err.Error()))
} else if err := instance.PostInit(ctx); err != nil {
return fmt.Errorf("error initializing configuration: %w", err)
}
if err := instance.PostInit(ctx); err != nil {
var migrationNeededErr *sqlite.MigrationNeededError
if errors.As(err, &migrationNeededErr) {
logger.Warn(err.Error())
} else {
panic(err)
return err
}
}
@@ -228,6 +252,9 @@ func initialize() error {
logger.Warnf("could not initialize FFMPEG subsystem: %v", err)
}
instance.Scanner = makeScanner(db, instance.PluginCache)
instance.Cleaner = makeCleaner(db, instance.PluginCache)
// if DLNA is enabled, start it now
if instance.Config.GetDLNADefaultEnabled() {
if err := instance.DLNAService.Start(nil); err != nil {
@@ -238,6 +265,71 @@ func initialize() error {
return nil
}
func videoFileFilter(f file.File) bool {
return isVideo(f.Base().Basename)
}
func imageFileFilter(f file.File) bool {
return isImage(f.Base().Basename)
}
func galleryFileFilter(f file.File) bool {
return isZip(f.Base().Basename)
}
type coverGenerator struct {
}
func (g *coverGenerator) GenerateCover(ctx context.Context, scene *models.Scene, f *file.VideoFile) error {
gg := generate.Generator{
Encoder: instance.FFMPEG,
LockManager: instance.ReadLockManager,
ScenePaths: instance.Paths.Scene,
}
return gg.Screenshot(ctx, f.Path, scene.GetHash(instance.Config.GetVideoFileNamingAlgorithm()), f.Width, f.Duration, generate.ScreenshotOptions{})
}
func makeScanner(db *sqlite.Database, pluginCache *plugin.Cache) *file.Scanner {
return &file.Scanner{
Repository: file.Repository{
Manager: db,
DatabaseProvider: db,
Store: db.File,
FolderStore: db.Folder,
},
FileDecorators: []file.Decorator{
&file.FilteredDecorator{
Decorator: &video.Decorator{
FFProbe: instance.FFProbe,
},
Filter: file.FilterFunc(videoFileFilter),
},
&file.FilteredDecorator{
Decorator: &file_image.Decorator{},
Filter: file.FilterFunc(imageFileFilter),
},
},
FingerprintCalculator: &fingerprintCalculator{instance.Config},
FS: &file.OsFS{},
}
}
func makeCleaner(db *sqlite.Database, pluginCache *plugin.Cache) *file.Cleaner {
return &file.Cleaner{
FS: &file.OsFS{},
Repository: file.Repository{
Manager: db,
DatabaseProvider: db,
Store: db.File,
FolderStore: db.Folder,
},
Handlers: []file.CleanHandler{
&cleanHandler{},
},
}
}
func initJobManager() *job.Manager {
ret := job.NewManager()
@@ -370,8 +462,12 @@ func (s *Manager) PostInit(ctx context.Context) error {
if err := fsutil.EmptyDir(instance.Paths.Generated.Downloads); err != nil {
logger.Warnf("could not empty Downloads directory: %v", err)
}
if err := fsutil.EmptyDir(instance.Paths.Generated.Tmp); err != nil {
logger.Warnf("could not empty Tmp directory: %v", err)
if err := fsutil.EnsureDir(instance.Paths.Generated.Tmp); err != nil {
logger.Warnf("could not create Tmp directory: %v", err)
} else {
if err := fsutil.EmptyDir(instance.Paths.Generated.Tmp); err != nil {
logger.Warnf("could not empty Tmp directory: %v", err)
}
}
}, deleteTimeout, func(done chan struct{}) {
logger.Info("Please wait. Deleting temporary files...") // print
@@ -526,6 +622,8 @@ func (s *Manager) Setup(ctx context.Context, input SetupInput) error {
return fmt.Errorf("error initializing FFMPEG subsystem: %v", err)
}
instance.Scanner = makeScanner(instance.Database, instance.PluginCache)
return nil
}