File storage rewrite (#2676)

* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes

pkg/file/file.go

@@ -1,31 +1,205 @@
 package file
 
 import (
+	"context"
 	"io"
 	"io/fs"
-	"os"
+	"net/http"
+	"strconv"
+	"time"
+
+	"github.com/stashapp/stash/pkg/logger"
 )
 
-type fsFile struct {
-	path string
-	info fs.FileInfo
+// ID represents an ID of a file.
+type ID int32
+
+func (i ID) String() string {
+	return strconv.Itoa(int(i))
 }
 
-func (f *fsFile) Open() (io.ReadCloser, error) {
-	return os.Open(f.path)
+// DirEntry represents a file or directory in the file system.
+type DirEntry struct {
+	ZipFileID *ID `json:"zip_file_id"`
+
+	// transient - not persisted
+	// only guaranteed to have id, path and basename set
+	ZipFile File
+
+	ModTime time.Time `json:"mod_time"`
 }
 
-func (f *fsFile) Path() string {
-	return f.path
+func (e *DirEntry) info(fs FS, path string) (fs.FileInfo, error) {
+	if e.ZipFile != nil {
+		zipPath := e.ZipFile.Base().Path
+		zfs, err := fs.OpenZip(zipPath)
+		if err != nil {
+			return nil, err
+		}
+		defer zfs.Close()
+		fs = zfs
+	}
+	// else assume os file
+
+	ret, err := fs.Lstat(path)
+	return ret, err
 }
 
-func (f *fsFile) FileInfo() fs.FileInfo {
-	return f.info
+// File represents a file in the file system.
+type File interface {
+	Base() *BaseFile
+	SetFingerprints(fp []Fingerprint)
+	Open(fs FS) (io.ReadCloser, error)
 }
 
-func FSFile(path string, info fs.FileInfo) SourceFile {
-	return &fsFile{
-		path: path,
-		info: info,
+// BaseFile represents a file in the file system.
+type BaseFile struct {
+	ID ID `json:"id"`
+
+	DirEntry
+
+	// resolved from parent folder and basename only - not stored in DB
+	Path string `json:"path"`
+
+	Basename       string   `json:"basename"`
+	ParentFolderID FolderID `json:"parent_folder_id"`
+
+	Fingerprints Fingerprints `json:"fingerprints"`
+
+	Size int64 `json:"size"`
+
+	CreatedAt time.Time `json:"created_at"`
+	UpdatedAt time.Time `json:"updated_at"`
+}
+
+// SetFingerprints sets the fingerprints of the file.
+// If a fingerprint of the same type already exists, it is overwritten.
+func (f *BaseFile) SetFingerprints(fp []Fingerprint) {
+	for _, v := range fp {
+		f.SetFingerprint(v)
 	}
 }
+
+// SetFingerprint sets the fingerprint of the file.
+// If a fingerprint of the same type already exists, it is overwritten.
+func (f *BaseFile) SetFingerprint(fp Fingerprint) {
+	for i, existing := range f.Fingerprints {
+		if existing.Type == fp.Type {
+			f.Fingerprints[i] = fp
+			return
+		}
+	}
+
+	f.Fingerprints = append(f.Fingerprints, fp)
+}
+
+// Base is used to fulfil the File interface.
+func (f *BaseFile) Base() *BaseFile {
+	return f
+}
+
+func (f *BaseFile) Open(fs FS) (io.ReadCloser, error) {
+	if f.ZipFile != nil {
+		zipPath := f.ZipFile.Base().Path
+		zfs, err := fs.OpenZip(zipPath)
+		if err != nil {
+			return nil, err
+		}
+
+		return zfs.OpenOnly(f.Path)
+	}
+
+	return fs.Open(f.Path)
+}
+
+func (f *BaseFile) Info(fs FS) (fs.FileInfo, error) {
+	return f.info(fs, f.Path)
+}
+
+func (f *BaseFile) Serve(fs FS, w http.ResponseWriter, r *http.Request) {
+	w.Header().Add("Cache-Control", "max-age=604800000") // 1 Week
+
+	reader, err := f.Open(fs)
+	if err != nil {
+		// assume not found
+		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
+		return
+	}
+
+	defer reader.Close()
+
+	rsc, ok := reader.(io.ReadSeeker)
+	if !ok {
+		// fallback to direct copy
+		data, err := io.ReadAll(reader)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+
+		if k, err := w.Write(data); err != nil {
+			logger.Warnf("failure while serving image (wrote %v bytes out of %v): %v", k, len(data), err)
+		}
+
+		return
+	}
+
+	http.ServeContent(w, r, f.Basename, f.ModTime, rsc)
+}
+
+type Finder interface {
+	Find(ctx context.Context, id ...ID) ([]File, error)
+}
+
+// Getter provides methods to find Files.
+type Getter interface {
+	FindByPath(ctx context.Context, path string) (File, error)
+	FindByFingerprint(ctx context.Context, fp Fingerprint) ([]File, error)
+	FindByZipFileID(ctx context.Context, zipFileID ID) ([]File, error)
+	FindAllInPaths(ctx context.Context, p []string, limit, offset int) ([]File, error)
+}
+
+type Counter interface {
+	CountAllInPaths(ctx context.Context, p []string) (int, error)
+}
+
+// Creator provides methods to create Files.
+type Creator interface {
+	Create(ctx context.Context, f File) error
+}
+
+// Updater provides methods to update Files.
+type Updater interface {
+	Update(ctx context.Context, f File) error
+}
+
+type Destroyer interface {
+	Destroy(ctx context.Context, id ID) error
+}
+
+// Store provides methods to find, create and update Files.
+type Store interface {
+	Getter
+	Counter
+	Creator
+	Updater
+	Destroyer
+}
+
+// Decorator wraps the Decorate method to add additional functionality while scanning files.
+type Decorator interface {
+	Decorate(ctx context.Context, fs FS, f File) (File, error)
+}
+
+type FilteredDecorator struct {
+	Decorator
+	Filter
+}
+
+// Decorate runs the decorator if the filter accepts the file.
+func (d *FilteredDecorator) Decorate(ctx context.Context, fs FS, f File) (File, error) {
+	if d.Accept(f) {
+		return d.Decorator.Decorate(ctx, fs, f)
+	}
+	return f, nil
+}