[Files Refactor] Import export fixup (#2763)

* Adjust json schema
* Remove mappings file from export
* Import file/folder support
* Update documentation
* Make gallery filenames unique

pkg/models/jsonschema/file_folder.go

@@ -0,0 +1,156 @@
+package jsonschema
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/stashapp/stash/pkg/hash/md5"
+	"github.com/stashapp/stash/pkg/models/json"
+)
+
+const (
+	DirEntryTypeFolder = "folder"
+	DirEntryTypeVideo  = "video"
+	DirEntryTypeImage  = "image"
+	DirEntryTypeFile   = "file"
+)
+
+type DirEntry interface {
+	IsFile() bool
+	Filename() string
+	DirEntry() *BaseDirEntry
+}
+
+type BaseDirEntry struct {
+	ZipFile string        `json:"zip_file,omitempty"`
+	ModTime json.JSONTime `json:"mod_time"`
+
+	Type string `json:"type,omitempty"`
+
+	Path string `json:"path,omitempty"`
+
+	CreatedAt json.JSONTime `json:"created_at,omitempty"`
+	UpdatedAt json.JSONTime `json:"updated_at,omitempty"`
+}
+
+func (f *BaseDirEntry) DirEntry() *BaseDirEntry {
+	return f
+}
+
+func (f *BaseDirEntry) IsFile() bool {
+	return false
+}
+
+func (f *BaseDirEntry) Filename() string {
+	// prefix with the path depth so that we can import lower-level files/folders first
+	depth := strings.Count(f.Path, string("/"))
+
+	// hash the full path for a unique filename
+	hash := md5.FromString(f.Path)
+
+	basename := path.Base(f.Path)
+
+	return fmt.Sprintf("%02x.%s.%s.json", depth, basename, hash)
+}
+
+type BaseFile struct {
+	BaseDirEntry
+
+	Fingerprints []Fingerprint `json:"fingerprints,omitempty"`
+	Size         int64         `json:"size"`
+}
+
+func (f *BaseFile) IsFile() bool {
+	return true
+}
+
+type Fingerprint struct {
+	Type        string      `json:"type,omitempty"`
+	Fingerprint interface{} `json:"fingerprint,omitempty"`
+}
+
+type VideoFile struct {
+	*BaseFile
+	Format     string  `json:"format,omitempty"`
+	Width      int     `json:"width,omitempty"`
+	Height     int     `json:"height,omitempty"`
+	Duration   float64 `json:"duration,omitempty"`
+	VideoCodec string  `json:"video_codec,omitempty"`
+	AudioCodec string  `json:"audio_codec,omitempty"`
+	FrameRate  float64 `json:"frame_rate,omitempty"`
+	BitRate    int64   `json:"bitrate,omitempty"`
+
+	Interactive      bool `json:"interactive,omitempty"`
+	InteractiveSpeed *int `json:"interactive_speed,omitempty"`
+}
+
+type ImageFile struct {
+	*BaseFile
+	Format string `json:"format,omitempty"`
+	Width  int    `json:"width,omitempty"`
+	Height int    `json:"height,omitempty"`
+}
+
+func LoadFileFile(filePath string) (DirEntry, error) {
+	r, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+
+	data, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	jsonParser := json.NewDecoder(bytes.NewReader(data))
+
+	var bf BaseDirEntry
+	if err := jsonParser.Decode(&bf); err != nil {
+		return nil, err
+	}
+
+	jsonParser = json.NewDecoder(bytes.NewReader(data))
+
+	switch bf.Type {
+	case DirEntryTypeFolder:
+		return &bf, nil
+	case DirEntryTypeVideo:
+		var vf VideoFile
+		if err := jsonParser.Decode(&vf); err != nil {
+			return nil, err
+		}
+
+		return &vf, nil
+	case DirEntryTypeImage:
+		var imf ImageFile
+		if err := jsonParser.Decode(&imf); err != nil {
+			return nil, err
+		}
+
+		return &imf, nil
+	case DirEntryTypeFile:
+		var bff BaseFile
+		if err := jsonParser.Decode(&bff); err != nil {
+			return nil, err
+		}
+
+		return &bff, nil
+	default:
+		return nil, errors.New("unknown file type")
+	}
+}
+
+func SaveFileFile(filePath string, file DirEntry) error {
+	if file == nil {
+		return fmt.Errorf("file must not be nil")
+	}
+	return marshalToFile(filePath, file)
+}

pkg/models/jsonschema/folder.go

@@ -0,0 +1,56 @@
+package jsonschema
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	"github.com/stashapp/stash/pkg/hash/md5"
+	"github.com/stashapp/stash/pkg/models/json"
+)
+
+type Folder struct {
+	BaseDirEntry
+
+	Path string `json:"path,omitempty"`
+
+	CreatedAt json.JSONTime `json:"created_at,omitempty"`
+	UpdatedAt json.JSONTime `json:"updated_at,omitempty"`
+}
+
+func (f *Folder) Filename() string {
+	// prefix with the path depth so that we can import lower-level folders first
+	depth := strings.Count(f.Path, string("/"))
+
+	// hash the full path for a unique filename
+	hash := md5.FromString(f.Path)
+
+	basename := path.Base(f.Path)
+
+	return fmt.Sprintf("%2x.%s.%s.json", depth, basename, hash)
+}
+
+func LoadFolderFile(filePath string) (*Folder, error) {
+	var folder Folder
+	file, err := os.Open(filePath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	jsonParser := json.NewDecoder(file)
+	err = jsonParser.Decode(&folder)
+	if err != nil {
+		return nil, err
+	}
+	return &folder, nil
+}
+
+func SaveFolderFile(filePath string, folder *Folder) error {
+	if folder == nil {
+		return fmt.Errorf("folder must not be nil")
+	}
+	return marshalToFile(filePath, folder)
+}

pkg/models/jsonschema/gallery.go

@@ -3,27 +3,37 @@ package jsonschema
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/stashapp/stash/pkg/models/json"
 )
 
 type Gallery struct {
-	Path        string        `json:"path,omitempty"`
-	Checksum    string        `json:"checksum,omitempty"`
-	Zip         bool          `json:"zip,omitempty"`
-	Title       string        `json:"title,omitempty"`
-	URL         string        `json:"url,omitempty"`
-	Date        string        `json:"date,omitempty"`
-	Details     string        `json:"details,omitempty"`
-	Rating      int           `json:"rating,omitempty"`
-	Organized   bool          `json:"organized,omitempty"`
-	Studio      string        `json:"studio,omitempty"`
-	Performers  []string      `json:"performers,omitempty"`
-	Tags        []string      `json:"tags,omitempty"`
-	FileModTime json.JSONTime `json:"file_mod_time,omitempty"`
-	CreatedAt   json.JSONTime `json:"created_at,omitempty"`
-	UpdatedAt   json.JSONTime `json:"updated_at,omitempty"`
+	ZipFiles   []string      `json:"zip_files,omitempty"`
+	FolderPath string        `json:"folder_path,omitempty"`
+	Title      string        `json:"title,omitempty"`
+	URL        string        `json:"url,omitempty"`
+	Date       string        `json:"date,omitempty"`
+	Details    string        `json:"details,omitempty"`
+	Rating     int           `json:"rating,omitempty"`
+	Organized  bool          `json:"organized,omitempty"`
+	Studio     string        `json:"studio,omitempty"`
+	Performers []string      `json:"performers,omitempty"`
+	Tags       []string      `json:"tags,omitempty"`
+	CreatedAt  json.JSONTime `json:"created_at,omitempty"`
+	UpdatedAt  json.JSONTime `json:"updated_at,omitempty"`
+}
+
+func (s Gallery) Filename(basename string, hash string) string {
+	ret := basename
+
+	if ret != "" {
+		ret += "."
+	}
+	ret += hash
+
+	return ret + ".json"
 }
 
 func LoadGalleryFile(filePath string) (*Gallery, error) {
@@ -48,3 +58,23 @@ func SaveGalleryFile(filePath string, gallery *Gallery) error {
 	}
 	return marshalToFile(filePath, gallery)
 }
+
+// GalleryRef is used to identify a Gallery.
+// Only one field should be populated.
+type GalleryRef struct {
+	ZipFiles   []string `json:"zip_files,omitempty"`
+	FolderPath string   `json:"folder_path,omitempty"`
+	// Title is used only if FolderPath and ZipPaths is empty
+	Title string `json:"title,omitempty"`
+}
+
+func (r GalleryRef) String() string {
+	switch {
+	case r.FolderPath != "":
+		return "{ folder: " + r.FolderPath + " }"
+	case len(r.ZipFiles) > 0:
+		return "{ zipFiles: [" + strings.Join(r.ZipFiles, ", ") + "] }"
+	default:
+		return "{ title: " + r.Title + " }"
+	}
+}

pkg/models/jsonschema/image.go

@@ -8,28 +8,33 @@ import (
 	"github.com/stashapp/stash/pkg/models/json"
 )
 
-type ImageFile struct {
-	ModTime json.JSONTime `json:"mod_time,omitempty"`
-	Size    int64         `json:"size"`
-	Width   int           `json:"width"`
-	Height  int           `json:"height"`
-}
-
 type Image struct {
 	Title      string        `json:"title,omitempty"`
-	Checksum   string        `json:"checksum,omitempty"`
 	Studio     string        `json:"studio,omitempty"`
 	Rating     int           `json:"rating,omitempty"`
 	Organized  bool          `json:"organized,omitempty"`
 	OCounter   int           `json:"o_counter,omitempty"`
-	Galleries  []string      `json:"galleries,omitempty"`
+	Galleries  []GalleryRef  `json:"galleries,omitempty"`
 	Performers []string      `json:"performers,omitempty"`
 	Tags       []string      `json:"tags,omitempty"`
-	File       *ImageFile    `json:"file,omitempty"`
+	Files      []string      `json:"files,omitempty"`
 	CreatedAt  json.JSONTime `json:"created_at,omitempty"`
 	UpdatedAt  json.JSONTime `json:"updated_at,omitempty"`
 }
 
+func (s Image) Filename(basename string, hash string) string {
+	ret := s.Title
+	if ret == "" {
+		ret = basename
+	}
+
+	if hash != "" {
+		ret += "." + hash
+	}
+
+	return ret + ".json"
+}
+
 func LoadImageFile(filePath string) (*Image, error) {
 	var image Image
 	file, err := os.Open(filePath)

pkg/models/jsonschema/mappings.go

@@ -1,47 +0,0 @@
-package jsonschema
-
-import (
-	"fmt"
-	"os"
-
-	jsoniter "github.com/json-iterator/go"
-)
-
-type PathNameMapping struct {
-	Path     string `json:"path,omitempty"`
-	Name     string `json:"name,omitempty"`
-	Checksum string `json:"checksum"`
-}
-
-type Mappings struct {
-	Tags       []PathNameMapping `json:"tags"`
-	Performers []PathNameMapping `json:"performers"`
-	Studios    []PathNameMapping `json:"studios"`
-	Movies     []PathNameMapping `json:"movies"`
-	Galleries  []PathNameMapping `json:"galleries"`
-	Scenes     []PathNameMapping `json:"scenes"`
-	Images     []PathNameMapping `json:"images"`
-}
-
-func LoadMappingsFile(filePath string) (*Mappings, error) {
-	var mappings Mappings
-	file, err := os.Open(filePath)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	jsonParser := json.NewDecoder(file)
-	err = jsonParser.Decode(&mappings)
-	if err != nil {
-		return nil, err
-	}
-	return &mappings, nil
-}
-
-func SaveMappingsFile(filePath string, mappings *Mappings) error {
-	if mappings == nil {
-		return fmt.Errorf("mappings must not be nil")
-	}
-	return marshalToFile(filePath, mappings)
-}

pkg/models/jsonschema/movie.go

@@ -26,6 +26,10 @@ type Movie struct {
 	UpdatedAt  json.JSONTime `json:"updated_at,omitempty"`
 }
 
+func (s Movie) Filename() string {
+	return s.Name + ".json"
+}
+
 // Backwards Compatible synopsis for the movie
 type MovieSynopsisBC struct {
 	Synopsis string `json:"sypnopsis,omitempty"`

pkg/models/jsonschema/performer.go

@@ -40,6 +40,10 @@ type Performer struct {
 	IgnoreAutoTag bool             `json:"ignore_auto_tag,omitempty"`
 }
 
+func (s Performer) Filename() string {
+	return s.Name + ".json"
+}
+
 func LoadPerformerFile(filePath string) (*Performer, error) {
 	var performer Performer
 	file, err := os.Open(filePath)

pkg/models/jsonschema/scene.go

@@ -38,9 +38,6 @@ type SceneMovie struct {
 
 type Scene struct {
 	Title      string           `json:"title,omitempty"`
-	Checksum   string           `json:"checksum,omitempty"`
-	OSHash     string           `json:"oshash,omitempty"`
-	Phash      string           `json:"phash,omitempty"`
 	Studio     string           `json:"studio,omitempty"`
 	URL        string           `json:"url,omitempty"`
 	Date       string           `json:"date,omitempty"`
@@ -48,18 +45,31 @@ type Scene struct {
 	Organized  bool             `json:"organized,omitempty"`
 	OCounter   int              `json:"o_counter,omitempty"`
 	Details    string           `json:"details,omitempty"`
-	Galleries  []string         `json:"galleries,omitempty"`
+	Galleries  []GalleryRef     `json:"galleries,omitempty"`
 	Performers []string         `json:"performers,omitempty"`
 	Movies     []SceneMovie     `json:"movies,omitempty"`
 	Tags       []string         `json:"tags,omitempty"`
 	Markers    []SceneMarker    `json:"markers,omitempty"`
-	File       *SceneFile       `json:"file,omitempty"`
+	Files      []string         `json:"files,omitempty"`
 	Cover      string           `json:"cover,omitempty"`
 	CreatedAt  json.JSONTime    `json:"created_at,omitempty"`
 	UpdatedAt  json.JSONTime    `json:"updated_at,omitempty"`
 	StashIDs   []models.StashID `json:"stash_ids,omitempty"`
 }
 
+func (s Scene) Filename(basename string, hash string) string {
+	ret := s.Title
+	if ret == "" {
+		ret = basename
+	}
+
+	if hash != "" {
+		ret += "." + hash
+	}
+
+	return ret + ".json"
+}
+
 func LoadSceneFile(filePath string) (*Scene, error) {
 	var scene Scene
 	file, err := os.Open(filePath)

pkg/models/jsonschema/studio.go

@@ -23,6 +23,10 @@ type Studio struct {
 	IgnoreAutoTag bool             `json:"ignore_auto_tag,omitempty"`
 }
 
+func (s Studio) Filename() string {
+	return s.Name + ".json"
+}
+
 func LoadStudioFile(filePath string) (*Studio, error) {
 	var studio Studio
 	file, err := os.Open(filePath)

pkg/models/jsonschema/tag.go

@@ -18,6 +18,10 @@ type Tag struct {
 	UpdatedAt     json.JSONTime `json:"updated_at,omitempty"`
 }
 
+func (s Tag) Filename() string {
+	return s.Name + ".json"
+}
+
 func LoadTagFile(filePath string) (*Tag, error) {
 	var tag Tag
 	file, err := os.Open(filePath)

pkg/models/paths/paths_json.go

@@ -10,8 +10,7 @@ import (
 type JSONPaths struct {
 	Metadata string
 
-	MappingsFile string
-	ScrapedFile  string
+	ScrapedFile string
 
 	Performers string
 	Scenes     string
@@ -20,12 +19,12 @@ type JSONPaths struct {
 	Studios    string
 	Tags       string
 	Movies     string
+	Files      string
 }
 
 func newJSONPaths(baseDir string) *JSONPaths {
 	jp := JSONPaths{}
 	jp.Metadata = baseDir
-	jp.MappingsFile = filepath.Join(baseDir, "mappings.json")
 	jp.ScrapedFile = filepath.Join(baseDir, "scraped.json")
 	jp.Performers = filepath.Join(baseDir, "performers")
 	jp.Scenes = filepath.Join(baseDir, "scenes")
@@ -34,6 +33,7 @@ func newJSONPaths(baseDir string) *JSONPaths {
 	jp.Studios = filepath.Join(baseDir, "studios")
 	jp.Movies = filepath.Join(baseDir, "movies")
 	jp.Tags = filepath.Join(baseDir, "tags")
+	jp.Files = filepath.Join(baseDir, "files")
 	return &jp
 }
 
@@ -42,6 +42,18 @@ func GetJSONPaths(baseDir string) *JSONPaths {
 	return jp
 }
 
+func EmptyJSONDirs(baseDir string) {
+	jsonPaths := GetJSONPaths(baseDir)
+	_ = fsutil.EmptyDir(jsonPaths.Scenes)
+	_ = fsutil.EmptyDir(jsonPaths.Images)
+	_ = fsutil.EmptyDir(jsonPaths.Galleries)
+	_ = fsutil.EmptyDir(jsonPaths.Performers)
+	_ = fsutil.EmptyDir(jsonPaths.Studios)
+	_ = fsutil.EmptyDir(jsonPaths.Movies)
+	_ = fsutil.EmptyDir(jsonPaths.Tags)
+	_ = fsutil.EmptyDir(jsonPaths.Files)
+}
+
 func EnsureJSONDirs(baseDir string) {
 	jsonPaths := GetJSONPaths(baseDir)
 	if err := fsutil.EnsureDir(jsonPaths.Metadata); err != nil {
@@ -68,32 +80,7 @@ func EnsureJSONDirs(baseDir string) {
 	if err := fsutil.EnsureDir(jsonPaths.Tags); err != nil {
 		logger.Warnf("couldn't create directories for Tags: %v", err)
 	}
-}
-
-func (jp *JSONPaths) PerformerJSONPath(checksum string) string {
-	return filepath.Join(jp.Performers, checksum+".json")
-}
-
-func (jp *JSONPaths) SceneJSONPath(checksum string) string {
-	return filepath.Join(jp.Scenes, checksum+".json")
-}
-
-func (jp *JSONPaths) ImageJSONPath(checksum string) string {
-	return filepath.Join(jp.Images, checksum+".json")
-}
-
-func (jp *JSONPaths) GalleryJSONPath(checksum string) string {
-	return filepath.Join(jp.Galleries, checksum+".json")
-}
-
-func (jp *JSONPaths) StudioJSONPath(checksum string) string {
-	return filepath.Join(jp.Studios, checksum+".json")
-}
-
-func (jp *JSONPaths) TagJSONPath(checksum string) string {
-	return filepath.Join(jp.Tags, checksum+".json")
-}
-
-func (jp *JSONPaths) MovieJSONPath(checksum string) string {
-	return filepath.Join(jp.Movies, checksum+".json")
+	if err := fsutil.EnsureDir(jsonPaths.Files); err != nil {
+		logger.Warnf("couldn't create directories for Files: %v", err)
+	}
 }