Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

[Files Refactor] Import export fixup (#2763)

Browse Source 
* Adjust json schema
* Remove mappings file from export
* Import file/folder support
* Update documentation
* Make gallery filenames unique
This commit is contained in:
WithoutPants
2022-08-30 12:17:15 +10:00
parent 1222b7b87b
commit 0b534d89c6
35 changed files with 3315 additions and 3146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
pkg/scene/import.go
View File

@@ -3,8 +3,10 @@ package scene
import (
"context"
"fmt"
"path/filepath"
"strings"
"github.com/stashapp/stash/pkg/file"
"github.com/stashapp/stash/pkg/gallery"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/models/jsonschema"
@@ -24,13 +26,13 @@ type FullCreatorUpdater interface {
type Importer struct {
ReaderWriter FullCreatorUpdater
FileFinder file.Getter
StudioWriter studio.NameFinderCreator
GalleryWriter gallery.ChecksumsFinder
GalleryFinder gallery.Finder
PerformerWriter performer.NameFinderCreator
MovieWriter movie.NameFinderCreator
TagWriter tag.NameFinderCreator
Input jsonschema.Scene
Path string
MissingRefBehaviour models.ImportMissingRefEnum
FileNamingAlgorithm models.HashAlgorithm
@@ -42,6 +44,10 @@ type Importer struct {
func (i *Importer) PreImport(ctx context.Context) error {
i.scene = i.sceneJSONToScene(i.Input)
if err := i.populateFiles(ctx); err != nil {
return err
}
if err := i.populateStudio(ctx); err != nil {
return err
}
@@ -86,21 +92,6 @@ func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
StashIDs: models.NewRelatedStashIDs(sceneJSON.StashIDs),
}
// if sceneJSON.Checksum != "" {
// newScene.Checksum = &sceneJSON.Checksum
// }
// if sceneJSON.OSHash != "" {
// newScene.OSHash = &sceneJSON.OSHash
// }
// if sceneJSON.Phash != "" {
// hash, err := strconv.ParseUint(sceneJSON.Phash, 16, 64)
// if err == nil {
// v := int64(hash)
// newScene.Phash = &v
// }
// }
if sceneJSON.Date != "" {
d := models.NewDate(sceneJSON.Date)
newScene.Date = &d
@@ -114,42 +105,27 @@ func (i *Importer) sceneJSONToScene(sceneJSON jsonschema.Scene) models.Scene {
newScene.CreatedAt = sceneJSON.CreatedAt.GetTime()
newScene.UpdatedAt = sceneJSON.UpdatedAt.GetTime()
// if sceneJSON.File != nil {
// if sceneJSON.File.Size != "" {
// newScene.Size = &sceneJSON.File.Size
// }
// if sceneJSON.File.Duration != "" {
// duration, _ := strconv.ParseFloat(sceneJSON.File.Duration, 64)
// newScene.Duration = &duration
// }
// if sceneJSON.File.VideoCodec != "" {
// newScene.VideoCodec = &sceneJSON.File.VideoCodec
// }
// if sceneJSON.File.AudioCodec != "" {
// newScene.AudioCodec = &sceneJSON.File.AudioCodec
// }
// if sceneJSON.File.Format != "" {
// newScene.Format = &sceneJSON.File.Format
// }
// if sceneJSON.File.Width != 0 {
// newScene.Width = &sceneJSON.File.Width
// }
// if sceneJSON.File.Height != 0 {
// newScene.Height = &sceneJSON.File.Height
// }
// if sceneJSON.File.Framerate != "" {
// framerate, _ := strconv.ParseFloat(sceneJSON.File.Framerate, 64)
// newScene.Framerate = &framerate
// }
// if sceneJSON.File.Bitrate != 0 {
// v := int64(sceneJSON.File.Bitrate)
// newScene.Bitrate = &v
// }
// }
return newScene
}
func (i *Importer) populateFiles(ctx context.Context) error {
for _, ref := range i.Input.Files {
path := filepath.FromSlash(ref)
f, err := i.FileFinder.FindByPath(ctx, path)
if err != nil {
return fmt.Errorf("error finding file: %w", err)
}
if f == nil {
return fmt.Errorf("scene file '%s' not found", path)
} else {
i.scene.Files = append(i.scene.Files, f.(*file.VideoFile))
}
}
return nil
}
func (i *Importer) populateStudio(ctx context.Context) error {
if i.Input.Studio != "" {
studio, err := i.StudioWriter.FindByName(ctx, i.Input.Studio, false)
@@ -192,33 +168,50 @@ func (i *Importer) createStudio(ctx context.Context, name string) (int, error) {
return created.ID, nil
}
func (i *Importer) locateGallery(ctx context.Context, ref jsonschema.GalleryRef) (*models.Gallery, error) {
var galleries []*models.Gallery
var err error
switch {
case ref.FolderPath != "":
galleries, err = i.GalleryFinder.FindByPath(ctx, ref.FolderPath)
case len(ref.ZipFiles) > 0:
for _, p := range ref.ZipFiles {
galleries, err = i.GalleryFinder.FindByPath(ctx, p)
if err != nil {
break
}
if len(galleries) > 0 {
break
}
}
case ref.Title != "":
galleries, err = i.GalleryFinder.FindUserGalleryByTitle(ctx, ref.Title)
}
var ret *models.Gallery
if len(galleries) > 0 {
ret = galleries[0]
}
return ret, err
}
func (i *Importer) populateGalleries(ctx context.Context) error {
if len(i.Input.Galleries) > 0 {
checksums := i.Input.Galleries
galleries, err := i.GalleryWriter.FindByChecksums(ctx, checksums)
for _, ref := range i.Input.Galleries {
gallery, err := i.locateGallery(ctx, ref)
if err != nil {
return err
}
var pluckedChecksums []string
for _, gallery := range galleries {
pluckedChecksums = append(pluckedChecksums, gallery.Checksum())
}
missingGalleries := stringslice.StrFilter(checksums, func(checksum string) bool {
return !stringslice.StrInclude(pluckedChecksums, checksum)
})
if len(missingGalleries) > 0 {
if gallery == nil {
if i.MissingRefBehaviour == models.ImportMissingRefEnumFail {
return fmt.Errorf("scene galleries [%s] not found", strings.Join(missingGalleries, ", "))
return fmt.Errorf("scene gallery '%s' not found", ref.String())
}
// we don't create galleries - just ignore
}
for _, o := range galleries {
i.scene.GalleryIDs.Add(o.ID)
} else {
i.scene.GalleryIDs.Add(gallery.ID)
}
}
@@ -366,37 +359,42 @@ func (i *Importer) PostImport(ctx context.Context, id int) error {
}
func (i *Importer) Name() string {
return i.Path
if i.Input.Title != "" {
return i.Input.Title
}
if len(i.Input.Files) > 0 {
return i.Input.Files[0]
}
return ""
}
func (i *Importer) FindExistingID(ctx context.Context) (*int, error) {
// TODO
// var existing []*models.Scene
// var err error
var existing []*models.Scene
var err error
// switch i.FileNamingAlgorithm {
// case models.HashAlgorithmMd5:
// existing, err = i.ReaderWriter.FindByChecksum(ctx, i.Input.Checksum)
// case models.HashAlgorithmOshash:
// existing, err = i.ReaderWriter.FindByOSHash(ctx, i.Input.OSHash)
// default:
// panic("unknown file naming algorithm")
// }
for _, f := range i.scene.Files {
existing, err = i.ReaderWriter.FindByFileID(ctx, f.ID)
if err != nil {
return nil, err
}
// if err != nil {
// return nil, err
// }
// if len(existing) > 0 {
// id := existing[0].ID
// return &id, nil
// }
if len(existing) > 0 {
id := existing[0].ID
return &id, nil
}
}
return nil, nil
}
func (i *Importer) Create(ctx context.Context) (*int, error) {
if err := i.ReaderWriter.Create(ctx, &i.scene, nil); err != nil {
var fileIDs []file.ID
for _, f := range i.scene.Files {
fileIDs = append(fileIDs, f.Base().ID)
}
if err := i.ReaderWriter.Create(ctx, &i.scene, fileIDs); err != nil {
return nil, fmt.Errorf("error creating scene: %v", err)
}