mirror of
https://github.com/stashapp/stash.git
synced 2025-12-18 04:44:37 +03:00
File storage rewrite (#2676)
* Restructure data layer part 2 (#2599) * Refactor and separate image model * Refactor image query builder * Handle relationships in image query builder * Remove relationship management methods * Refactor gallery model/query builder * Add scenes to gallery model * Convert scene model * Refactor scene models * Remove unused methods * Add unit tests for gallery * Add image tests * Add scene tests * Convert unnecessary scene value pointers to values * Convert unnecessary pointer values to values * Refactor scene partial * Add scene partial tests * Refactor ImagePartial * Add image partial tests * Refactor gallery partial update * Add partial gallery update tests * Use zero/null package for null values * Add files and scan system * Add sqlite implementation for files/folders * Add unit tests for files/folders * Image refactors * Update image data layer * Refactor gallery model and creation * Refactor scene model * Refactor scenes * Don't set title from filename * Allow galleries to freely add/remove images * Add multiple scene file support to graphql and UI * Add multiple file support for images in graphql/UI * Add multiple file for galleries in graphql/UI * Remove use of some deprecated fields * Remove scene path usage * Remove gallery path usage * Remove path from image * Move funscript to video file * Refactor caption detection * Migrate existing data * Add post commit/rollback hook system * Lint. Comment out import/export tests * Add WithDatabase read only wrapper * Prepend tasks to list * Add 32 pre-migration * Add warnings in release and migration notes
This commit is contained in:
WithoutPants
@@ -32,7 +32,7 @@ import (
|
||||
)
|
||||
|
||||
type ExportTask struct {
|
||||
txnManager models.Repository
|
||||
txnManager Repository
|
||||
full bool
|
||||
|
||||
baseDir string
|
||||
@@ -286,7 +286,7 @@ func (t *ExportTask) zipFile(fn, outDir string, z *zip.Writer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *ExportTask) populateMovieScenes(ctx context.Context, repo models.Repository) {
|
||||
func (t *ExportTask) populateMovieScenes(ctx context.Context, repo Repository) {
|
||||
reader := repo.Movie
|
||||
sceneReader := repo.Scene
|
||||
|
||||
@@ -316,7 +316,7 @@ func (t *ExportTask) populateMovieScenes(ctx context.Context, repo models.Reposi
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) populateGalleryImages(ctx context.Context, repo models.Repository) {
|
||||
func (t *ExportTask) populateGalleryImages(ctx context.Context, repo Repository) {
|
||||
reader := repo.Gallery
|
||||
imageReader := repo.Image
|
||||
|
||||
@@ -346,7 +346,7 @@ func (t *ExportTask) populateGalleryImages(ctx context.Context, repo models.Repo
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportScenes(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportScenes(ctx context.Context, workers int, repo Repository) {
|
||||
var scenesWg sync.WaitGroup
|
||||
|
||||
sceneReader := repo.Scene
|
||||
@@ -380,7 +380,7 @@ func (t *ExportTask) ExportScenes(ctx context.Context, workers int, repo models.
|
||||
if (i % 100) == 0 { // make progress easier to read
|
||||
logger.Progressf("[scenes] %d of %d", index, len(scenes))
|
||||
}
|
||||
t.Mappings.Scenes = append(t.Mappings.Scenes, jsonschema.PathNameMapping{Path: scene.Path, Checksum: scene.GetHash(t.fileNamingAlgorithm)})
|
||||
t.Mappings.Scenes = append(t.Mappings.Scenes, jsonschema.PathNameMapping{Path: scene.Path(), Checksum: scene.GetHash(t.fileNamingAlgorithm)})
|
||||
jobCh <- scene // feed workers
|
||||
}
|
||||
|
||||
@@ -390,7 +390,7 @@ func (t *ExportTask) ExportScenes(ctx context.Context, workers int, repo models.
|
||||
logger.Infof("[scenes] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func exportScene(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Scene, repo models.Repository, t *ExportTask) {
|
||||
func exportScene(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Scene, repo Repository, t *ExportTask) {
|
||||
defer wg.Done()
|
||||
sceneReader := repo.Scene
|
||||
studioReader := repo.Studio
|
||||
@@ -443,15 +443,15 @@ func exportScene(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
continue
|
||||
}
|
||||
|
||||
newSceneJSON.Movies, err = scene.GetSceneMoviesJSON(ctx, movieReader, sceneReader, s)
|
||||
newSceneJSON.Movies, err = scene.GetSceneMoviesJSON(ctx, movieReader, s)
|
||||
if err != nil {
|
||||
logger.Errorf("[scenes] <%s> error getting scene movies JSON: %s", sceneHash, err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if t.includeDependencies {
|
||||
if s.StudioID.Valid {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64))
|
||||
if s.StudioID != nil {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, *s.StudioID)
|
||||
}
|
||||
|
||||
t.galleries.IDs = intslice.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(galleries))
|
||||
@@ -463,7 +463,7 @@ func exportScene(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
}
|
||||
t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tagIDs)
|
||||
|
||||
movieIDs, err := scene.GetDependentMovieIDs(ctx, sceneReader, s)
|
||||
movieIDs, err := scene.GetDependentMovieIDs(ctx, s)
|
||||
if err != nil {
|
||||
logger.Errorf("[scenes] <%s> error getting scene movies: %s", sceneHash, err.Error())
|
||||
continue
|
||||
@@ -484,7 +484,7 @@ func exportScene(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportImages(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportImages(ctx context.Context, workers int, repo Repository) {
|
||||
var imagesWg sync.WaitGroup
|
||||
|
||||
imageReader := repo.Image
|
||||
@@ -518,7 +518,7 @@ func (t *ExportTask) ExportImages(ctx context.Context, workers int, repo models.
|
||||
if (i % 100) == 0 { // make progress easier to read
|
||||
logger.Progressf("[images] %d of %d", index, len(images))
|
||||
}
|
||||
t.Mappings.Images = append(t.Mappings.Images, jsonschema.PathNameMapping{Path: image.Path, Checksum: image.Checksum})
|
||||
t.Mappings.Images = append(t.Mappings.Images, jsonschema.PathNameMapping{Path: image.Path(), Checksum: image.Checksum()})
|
||||
jobCh <- image // feed workers
|
||||
}
|
||||
|
||||
@@ -528,7 +528,7 @@ func (t *ExportTask) ExportImages(ctx context.Context, workers int, repo models.
|
||||
logger.Infof("[images] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Image, repo models.Repository, t *ExportTask) {
|
||||
func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Image, repo Repository, t *ExportTask) {
|
||||
defer wg.Done()
|
||||
studioReader := repo.Studio
|
||||
galleryReader := repo.Gallery
|
||||
@@ -536,7 +536,7 @@ func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
tagReader := repo.Tag
|
||||
|
||||
for s := range jobChan {
|
||||
imageHash := s.Checksum
|
||||
imageHash := s.Checksum()
|
||||
|
||||
newImageJSON := image.ToBasicJSON(s)
|
||||
|
||||
@@ -572,8 +572,8 @@ func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
newImageJSON.Tags = tag.GetNames(tags)
|
||||
|
||||
if t.includeDependencies {
|
||||
if s.StudioID.Valid {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(s.StudioID.Int64))
|
||||
if s.StudioID != nil {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, *s.StudioID)
|
||||
}
|
||||
|
||||
t.galleries.IDs = intslice.IntAppendUniques(t.galleries.IDs, gallery.GetIDs(imageGalleries))
|
||||
@@ -594,12 +594,12 @@ func exportImage(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models
|
||||
|
||||
func (t *ExportTask) getGalleryChecksums(galleries []*models.Gallery) (ret []string) {
|
||||
for _, g := range galleries {
|
||||
ret = append(ret, g.Checksum)
|
||||
ret = append(ret, g.Checksum())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportGalleries(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportGalleries(ctx context.Context, workers int, repo Repository) {
|
||||
var galleriesWg sync.WaitGroup
|
||||
|
||||
reader := repo.Gallery
|
||||
@@ -634,10 +634,13 @@ func (t *ExportTask) ExportGalleries(ctx context.Context, workers int, repo mode
|
||||
logger.Progressf("[galleries] %d of %d", index, len(galleries))
|
||||
}
|
||||
|
||||
title := gallery.Title
|
||||
path := gallery.Path()
|
||||
|
||||
t.Mappings.Galleries = append(t.Mappings.Galleries, jsonschema.PathNameMapping{
|
||||
Path: gallery.Path.String,
|
||||
Name: gallery.Title.String,
|
||||
Checksum: gallery.Checksum,
|
||||
Path: path,
|
||||
Name: title,
|
||||
Checksum: gallery.Checksum(),
|
||||
})
|
||||
jobCh <- gallery
|
||||
}
|
||||
@@ -648,14 +651,14 @@ func (t *ExportTask) ExportGalleries(ctx context.Context, workers int, repo mode
|
||||
logger.Infof("[galleries] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func exportGallery(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Gallery, repo models.Repository, t *ExportTask) {
|
||||
func exportGallery(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Gallery, repo Repository, t *ExportTask) {
|
||||
defer wg.Done()
|
||||
studioReader := repo.Studio
|
||||
performerReader := repo.Performer
|
||||
tagReader := repo.Tag
|
||||
|
||||
for g := range jobChan {
|
||||
galleryHash := g.Checksum
|
||||
galleryHash := g.Checksum()
|
||||
|
||||
newGalleryJSON, err := gallery.ToBasicJSON(g)
|
||||
if err != nil {
|
||||
@@ -686,8 +689,8 @@ func exportGallery(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *mode
|
||||
newGalleryJSON.Tags = tag.GetNames(tags)
|
||||
|
||||
if t.includeDependencies {
|
||||
if g.StudioID.Valid {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, int(g.StudioID.Int64))
|
||||
if g.StudioID != nil {
|
||||
t.studios.IDs = intslice.IntAppendUnique(t.studios.IDs, *g.StudioID)
|
||||
}
|
||||
|
||||
t.tags.IDs = intslice.IntAppendUniques(t.tags.IDs, tag.GetIDs(tags))
|
||||
@@ -705,7 +708,7 @@ func exportGallery(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *mode
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportPerformers(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportPerformers(ctx context.Context, workers int, repo Repository) {
|
||||
var performersWg sync.WaitGroup
|
||||
|
||||
reader := repo.Performer
|
||||
@@ -745,7 +748,7 @@ func (t *ExportTask) ExportPerformers(ctx context.Context, workers int, repo mod
|
||||
logger.Infof("[performers] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func (t *ExportTask) exportPerformer(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Performer, repo models.Repository) {
|
||||
func (t *ExportTask) exportPerformer(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Performer, repo Repository) {
|
||||
defer wg.Done()
|
||||
|
||||
performerReader := repo.Performer
|
||||
@@ -783,7 +786,7 @@ func (t *ExportTask) exportPerformer(ctx context.Context, wg *sync.WaitGroup, jo
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportStudios(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportStudios(ctx context.Context, workers int, repo Repository) {
|
||||
var studiosWg sync.WaitGroup
|
||||
|
||||
reader := repo.Studio
|
||||
@@ -824,7 +827,7 @@ func (t *ExportTask) ExportStudios(ctx context.Context, workers int, repo models
|
||||
logger.Infof("[studios] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func (t *ExportTask) exportStudio(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Studio, repo models.Repository) {
|
||||
func (t *ExportTask) exportStudio(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Studio, repo Repository) {
|
||||
defer wg.Done()
|
||||
|
||||
studioReader := repo.Studio
|
||||
@@ -848,7 +851,7 @@ func (t *ExportTask) exportStudio(ctx context.Context, wg *sync.WaitGroup, jobCh
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportTags(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportTags(ctx context.Context, workers int, repo Repository) {
|
||||
var tagsWg sync.WaitGroup
|
||||
|
||||
reader := repo.Tag
|
||||
@@ -892,7 +895,7 @@ func (t *ExportTask) ExportTags(ctx context.Context, workers int, repo models.Re
|
||||
logger.Infof("[tags] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
}
|
||||
|
||||
func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Tag, repo models.Repository) {
|
||||
func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Tag, repo Repository) {
|
||||
defer wg.Done()
|
||||
|
||||
tagReader := repo.Tag
|
||||
@@ -919,7 +922,7 @@ func (t *ExportTask) exportTag(ctx context.Context, wg *sync.WaitGroup, jobChan
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportMovies(ctx context.Context, workers int, repo models.Repository) {
|
||||
func (t *ExportTask) ExportMovies(ctx context.Context, workers int, repo Repository) {
|
||||
var moviesWg sync.WaitGroup
|
||||
|
||||
reader := repo.Movie
|
||||
@@ -960,7 +963,7 @@ func (t *ExportTask) ExportMovies(ctx context.Context, workers int, repo models.
|
||||
logger.Infof("[movies] export complete in %s. %d workers used.", time.Since(startTime), workers)
|
||||
|
||||
}
|
||||
func (t *ExportTask) exportMovie(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Movie, repo models.Repository) {
|
||||
func (t *ExportTask) exportMovie(ctx context.Context, wg *sync.WaitGroup, jobChan <-chan *models.Movie, repo Repository) {
|
||||
defer wg.Done()
|
||||
|
||||
movieReader := repo.Movie
|
||||
@@ -993,7 +996,7 @@ func (t *ExportTask) exportMovie(ctx context.Context, wg *sync.WaitGroup, jobCha
|
||||
}
|
||||
}
|
||||
|
||||
func (t *ExportTask) ExportScrapedItems(ctx context.Context, repo models.Repository) {
|
||||
func (t *ExportTask) ExportScrapedItems(ctx context.Context, repo Repository) {
|
||||
qb := repo.ScrapedItem
|
||||
sqb := repo.Studio
|
||||
scrapedItems, err := qb.All(ctx)
|
||||
|
||||
Reference in New Issue
Block a user