mirror of
https://github.com/stashapp/stash.git
synced 2025-12-18 04:44:37 +03:00
File storage rewrite (#2676)
* Restructure data layer part 2 (#2599) * Refactor and separate image model * Refactor image query builder * Handle relationships in image query builder * Remove relationship management methods * Refactor gallery model/query builder * Add scenes to gallery model * Convert scene model * Refactor scene models * Remove unused methods * Add unit tests for gallery * Add image tests * Add scene tests * Convert unnecessary scene value pointers to values * Convert unnecessary pointer values to values * Refactor scene partial * Add scene partial tests * Refactor ImagePartial * Add image partial tests * Refactor gallery partial update * Add partial gallery update tests * Use zero/null package for null values * Add files and scan system * Add sqlite implementation for files/folders * Add unit tests for files/folders * Image refactors * Update image data layer * Refactor gallery model and creation * Refactor scene model * Refactor scenes * Don't set title from filename * Allow galleries to freely add/remove images * Add multiple scene file support to graphql and UI * Add multiple file support for images in graphql/UI * Add multiple file for galleries in graphql/UI * Remove use of some deprecated fields * Remove scene path usage * Remove gallery path usage * Remove path from image * Move funscript to video file * Refactor caption detection * Migrate existing data * Add post commit/rollback hook system * Lint. Comment out import/export tests * Add WithDatabase read only wrapper * Prepend tasks to list * Add 32 pre-migration * Add warnings in release and migration notes
This commit is contained in:
WithoutPants
@@ -1,170 +1,160 @@
|
||||
package manager
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
// func (t *ScanTask) scanGallery(ctx context.Context) {
|
||||
// var g *models.Gallery
|
||||
// path := t.file.Path()
|
||||
// images := 0
|
||||
// scanImages := false
|
||||
|
||||
"github.com/remeh/sizedwaitgroup"
|
||||
"github.com/stashapp/stash/internal/manager/config"
|
||||
"github.com/stashapp/stash/pkg/file"
|
||||
"github.com/stashapp/stash/pkg/gallery"
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
)
|
||||
// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
// var err error
|
||||
// g, err = t.TxnManager.Gallery.FindByPath(ctx, path)
|
||||
|
||||
func (t *ScanTask) scanGallery(ctx context.Context) {
|
||||
var g *models.Gallery
|
||||
path := t.file.Path()
|
||||
images := 0
|
||||
scanImages := false
|
||||
// if g != nil && err == nil {
|
||||
// images, err = t.TxnManager.Image.CountByGalleryID(ctx, g.ID)
|
||||
// if err != nil {
|
||||
// return fmt.Errorf("error getting images for zip gallery %s: %s", path, err.Error())
|
||||
// }
|
||||
// }
|
||||
|
||||
if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
var err error
|
||||
g, err = t.TxnManager.Gallery.FindByPath(ctx, path)
|
||||
// return err
|
||||
// }); err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
if g != nil && err == nil {
|
||||
images, err = t.TxnManager.Image.CountByGalleryID(ctx, g.ID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting images for zip gallery %s: %s", path, err.Error())
|
||||
}
|
||||
}
|
||||
// scanner := gallery.Scanner{
|
||||
// Scanner: gallery.FileScanner(&file.FSHasher{}),
|
||||
// ImageExtensions: instance.Config.GetImageExtensions(),
|
||||
// StripFileExtension: t.StripFileExtension,
|
||||
// CaseSensitiveFs: t.CaseSensitiveFs,
|
||||
// CreatorUpdater: t.TxnManager.Gallery,
|
||||
// Paths: instance.Paths,
|
||||
// PluginCache: instance.PluginCache,
|
||||
// MutexManager: t.mutexManager,
|
||||
// }
|
||||
|
||||
return err
|
||||
}); err != nil {
|
||||
logger.Error(err.Error())
|
||||
return
|
||||
}
|
||||
// var err error
|
||||
// if g != nil {
|
||||
// g, scanImages, err = scanner.ScanExisting(ctx, g, t.file)
|
||||
// if err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
scanner := gallery.Scanner{
|
||||
Scanner: gallery.FileScanner(&file.FSHasher{}),
|
||||
ImageExtensions: instance.Config.GetImageExtensions(),
|
||||
StripFileExtension: t.StripFileExtension,
|
||||
CaseSensitiveFs: t.CaseSensitiveFs,
|
||||
CreatorUpdater: t.TxnManager.Gallery,
|
||||
Paths: instance.Paths,
|
||||
PluginCache: instance.PluginCache,
|
||||
MutexManager: t.mutexManager,
|
||||
}
|
||||
// // scan the zip files if the gallery has no images
|
||||
// scanImages = scanImages || images == 0
|
||||
// } else {
|
||||
// g, scanImages, err = scanner.ScanNew(ctx, t.file)
|
||||
// if err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// }
|
||||
// }
|
||||
|
||||
var err error
|
||||
if g != nil {
|
||||
g, scanImages, err = scanner.ScanExisting(ctx, g, t.file)
|
||||
if err != nil {
|
||||
logger.Error(err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// scan the zip files if the gallery has no images
|
||||
scanImages = scanImages || images == 0
|
||||
} else {
|
||||
g, scanImages, err = scanner.ScanNew(ctx, t.file)
|
||||
if err != nil {
|
||||
logger.Error(err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
if g != nil {
|
||||
if scanImages {
|
||||
t.scanZipImages(ctx, g)
|
||||
} else {
|
||||
// in case thumbnails have been deleted, regenerate them
|
||||
t.regenerateZipImages(ctx, g)
|
||||
}
|
||||
}
|
||||
}
|
||||
// if g != nil {
|
||||
// if scanImages {
|
||||
// t.scanZipImages(ctx, g)
|
||||
// } else {
|
||||
// // in case thumbnails have been deleted, regenerate them
|
||||
// t.regenerateZipImages(ctx, g)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// associates a gallery to a scene with the same basename
|
||||
func (t *ScanTask) associateGallery(ctx context.Context, wg *sizedwaitgroup.SizedWaitGroup) {
|
||||
path := t.file.Path()
|
||||
if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
r := t.TxnManager
|
||||
qb := r.Gallery
|
||||
sqb := r.Scene
|
||||
g, err := qb.FindByPath(ctx, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// func (t *ScanTask) associateGallery(ctx context.Context, wg *sizedwaitgroup.SizedWaitGroup) {
|
||||
// path := t.file.Path()
|
||||
// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
// r := t.TxnManager
|
||||
// qb := r.Gallery
|
||||
// sqb := r.Scene
|
||||
// g, err := qb.FindByPath(ctx, path)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
|
||||
if g == nil {
|
||||
// associate is run after scan is finished
|
||||
// should only happen if gallery is a directory or an io error occurs during hashing
|
||||
logger.Warnf("associate: gallery %s not found in DB", path)
|
||||
return nil
|
||||
}
|
||||
// if g == nil {
|
||||
// // associate is run after scan is finished
|
||||
// // should only happen if gallery is a directory or an io error occurs during hashing
|
||||
// logger.Warnf("associate: gallery %s not found in DB", path)
|
||||
// return nil
|
||||
// }
|
||||
|
||||
basename := strings.TrimSuffix(path, filepath.Ext(path))
|
||||
var relatedFiles []string
|
||||
vExt := config.GetInstance().GetVideoExtensions()
|
||||
// make a list of media files that can be related to the gallery
|
||||
for _, ext := range vExt {
|
||||
related := basename + "." + ext
|
||||
// exclude gallery extensions from the related files
|
||||
if !isGallery(related) {
|
||||
relatedFiles = append(relatedFiles, related)
|
||||
}
|
||||
}
|
||||
for _, scenePath := range relatedFiles {
|
||||
scene, _ := sqb.FindByPath(ctx, scenePath)
|
||||
// found related Scene
|
||||
if scene != nil {
|
||||
sceneGalleries, _ := sqb.FindByGalleryID(ctx, g.ID) // check if gallery is already associated to the scene
|
||||
isAssoc := false
|
||||
for _, sg := range sceneGalleries {
|
||||
if scene.ID == sg.ID {
|
||||
isAssoc = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !isAssoc {
|
||||
logger.Infof("associate: Gallery %s is related to scene: %d", path, scene.ID)
|
||||
if err := sqb.UpdateGalleries(ctx, scene.ID, []int{g.ID}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
logger.Error(err.Error())
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
// basename := strings.TrimSuffix(path, filepath.Ext(path))
|
||||
// var relatedFiles []string
|
||||
// vExt := config.GetInstance().GetVideoExtensions()
|
||||
// // make a list of media files that can be related to the gallery
|
||||
// for _, ext := range vExt {
|
||||
// related := basename + "." + ext
|
||||
// // exclude gallery extensions from the related files
|
||||
// if !isGallery(related) {
|
||||
// relatedFiles = append(relatedFiles, related)
|
||||
// }
|
||||
// }
|
||||
// for _, scenePath := range relatedFiles {
|
||||
// scene, _ := sqb.FindByPath(ctx, scenePath)
|
||||
// // found related Scene
|
||||
// if scene != nil {
|
||||
// sceneGalleries, _ := sqb.FindByGalleryID(ctx, g.ID) // check if gallery is already associated to the scene
|
||||
// isAssoc := false
|
||||
// for _, sg := range sceneGalleries {
|
||||
// if scene.ID == sg.ID {
|
||||
// isAssoc = true
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if !isAssoc {
|
||||
// logger.Infof("associate: Gallery %s is related to scene: %d", path, scene.ID)
|
||||
// if _, err := sqb.UpdatePartial(ctx, scene.ID, models.ScenePartial{
|
||||
// GalleryIDs: &models.UpdateIDs{
|
||||
// IDs: []int{g.ID},
|
||||
// Mode: models.RelationshipUpdateModeAdd,
|
||||
// },
|
||||
// }); err != nil {
|
||||
// return err
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return nil
|
||||
// }); err != nil {
|
||||
// logger.Error(err.Error())
|
||||
// }
|
||||
// wg.Done()
|
||||
// }
|
||||
|
||||
func (t *ScanTask) scanZipImages(ctx context.Context, zipGallery *models.Gallery) {
|
||||
err := walkGalleryZip(zipGallery.Path.String, func(f *zip.File) error {
|
||||
// copy this task and change the filename
|
||||
subTask := *t
|
||||
// func (t *ScanTask) scanZipImages(ctx context.Context, zipGallery *models.Gallery) {
|
||||
// err := walkGalleryZip(*zipGallery.Path, func(f *zip.File) error {
|
||||
// // copy this task and change the filename
|
||||
// subTask := *t
|
||||
|
||||
// filepath is the zip file and the internal file name, separated by a null byte
|
||||
subTask.file = file.ZipFile(zipGallery.Path.String, f)
|
||||
subTask.zipGallery = zipGallery
|
||||
// // filepath is the zip file and the internal file name, separated by a null byte
|
||||
// subTask.file = file.ZipFile(*zipGallery.Path, f)
|
||||
// subTask.zipGallery = zipGallery
|
||||
|
||||
// run the subtask and wait for it to complete
|
||||
subTask.Start(ctx)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
logger.Warnf("failed to scan zip file images for %s: %s", zipGallery.Path.String, err.Error())
|
||||
}
|
||||
}
|
||||
// // run the subtask and wait for it to complete
|
||||
// subTask.Start(ctx)
|
||||
// return nil
|
||||
// })
|
||||
// if err != nil {
|
||||
// logger.Warnf("failed to scan zip file images for %s: %s", *zipGallery.Path, err.Error())
|
||||
// }
|
||||
// }
|
||||
|
||||
func (t *ScanTask) regenerateZipImages(ctx context.Context, zipGallery *models.Gallery) {
|
||||
var images []*models.Image
|
||||
if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
iqb := t.TxnManager.Image
|
||||
// func (t *ScanTask) regenerateZipImages(ctx context.Context, zipGallery *models.Gallery) {
|
||||
// var images []*models.Image
|
||||
// if err := t.TxnManager.WithTxn(ctx, func(ctx context.Context) error {
|
||||
// iqb := t.TxnManager.Image
|
||||
|
||||
var err error
|
||||
images, err = iqb.FindByGalleryID(ctx, zipGallery.ID)
|
||||
return err
|
||||
}); err != nil {
|
||||
logger.Warnf("failed to find gallery images: %s", err.Error())
|
||||
return
|
||||
}
|
||||
// var err error
|
||||
// images, err = iqb.FindByGalleryID(ctx, zipGallery.ID)
|
||||
// return err
|
||||
// }); err != nil {
|
||||
// logger.Warnf("failed to find gallery images: %s", err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
for _, img := range images {
|
||||
t.generateThumbnail(img)
|
||||
}
|
||||
}
|
||||
// for _, img := range images {
|
||||
// t.generateThumbnail(img)
|
||||
// }
|
||||
// }
|
||||
|
||||
Reference in New Issue
Block a user