File storage rewrite (#2676)

* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes

pkg/file/zip.go

@@ -2,63 +2,135 @@ package file
 
 import (
 	"archive/zip"
+	"errors"
+	"fmt"
 	"io"
 	"io/fs"
-	"strings"
+	"path/filepath"
 )
 
-const zipSeparator = "\x00"
+var (
+	errNotReaderAt  = errors.New("not a ReaderAt")
+	errZipFSOpenZip = errors.New("cannot open zip file inside zip file")
+)
 
-type zipFile struct {
-	zipPath string
-	zf      *zip.File
+// ZipFS is a file system backed by a zip file.
+type ZipFS struct {
+	*zip.Reader
+	zipFileCloser io.Closer
+	zipInfo       fs.FileInfo
+	zipPath       string
 }
 
-func (f *zipFile) Open() (io.ReadCloser, error) {
-	return f.zf.Open()
-}
-
-func (f *zipFile) Path() string {
-	// TODO - fix this
-	return ZipFilename(f.zipPath, f.zf.Name)
-}
-
-func (f *zipFile) FileInfo() fs.FileInfo {
-	return f.zf.FileInfo()
-}
-
-func ZipFile(zipPath string, zf *zip.File) SourceFile {
-	return &zipFile{
-		zipPath: zipPath,
-		zf:      zf,
+func newZipFS(fs FS, path string, info fs.FileInfo) (*ZipFS, error) {
+	reader, err := fs.Open(path)
+	if err != nil {
+		return nil, err
 	}
-}
 
-func ZipFilename(zipFilename, filenameInZip string) string {
-	return zipFilename + zipSeparator + filenameInZip
-}
-
-// IsZipPath returns true if the path includes the zip separator byte,
-// indicating it is within a zip file.
-func IsZipPath(p string) bool {
-	return strings.Contains(p, zipSeparator)
-}
-
-// ZipPathDisplayName converts an zip path for display. It translates the zip
-// file separator character into '/', since this character is also used for
-// path separators within zip files. It returns the original provided path
-// if it does not contain the zip file separator character.
-func ZipPathDisplayName(path string) string {
-	return strings.ReplaceAll(path, zipSeparator, "/")
-}
-
-func ZipFilePath(path string) (zipFilename, filename string) {
-	nullIndex := strings.Index(path, zipSeparator)
-	if nullIndex != -1 {
-		zipFilename = path[0:nullIndex]
-		filename = path[nullIndex+1:]
-	} else {
-		filename = path
+	asReaderAt, _ := reader.(io.ReaderAt)
+	if asReaderAt == nil {
+		reader.Close()
+		return nil, errNotReaderAt
 	}
-	return
+
+	zipReader, err := zip.NewReader(asReaderAt, info.Size())
+	if err != nil {
+		reader.Close()
+		return nil, err
+	}
+
+	return &ZipFS{
+		Reader:        zipReader,
+		zipFileCloser: reader,
+		zipInfo:       info,
+		zipPath:       path,
+	}, nil
+}
+
+func (f *ZipFS) rel(name string) (string, error) {
+	if f.zipPath == name {
+		return ".", nil
+	}
+
+	relName, err := filepath.Rel(f.zipPath, name)
+	if err != nil {
+		return "", fmt.Errorf("internal error getting relative path: %w", err)
+	}
+
+	// convert relName to use slash, since zip files do so regardless
+	// of os
+	relName = filepath.ToSlash(relName)
+
+	return relName, nil
+}
+
+func (f *ZipFS) Lstat(name string) (fs.FileInfo, error) {
+	reader, err := f.Open(name)
+	if err != nil {
+		return nil, err
+	}
+	defer reader.Close()
+
+	return reader.Stat()
+}
+
+func (f *ZipFS) OpenZip(name string) (*ZipFS, error) {
+	return nil, errZipFSOpenZip
+}
+
+type zipReadDirFile struct {
+	fs.File
+}
+
+func (f *zipReadDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	asReadDirFile, _ := f.File.(fs.ReadDirFile)
+	if asReadDirFile == nil {
+		return nil, fmt.Errorf("internal error: not a ReadDirFile")
+	}
+
+	return asReadDirFile.ReadDir(n)
+}
+
+func (f *ZipFS) Open(name string) (fs.ReadDirFile, error) {
+	relName, err := f.rel(name)
+	if err != nil {
+		return nil, err
+	}
+
+	r, err := f.Reader.Open(relName)
+	if err != nil {
+		return nil, err
+	}
+
+	return &zipReadDirFile{
+		File: r,
+	}, nil
+}
+
+func (f *ZipFS) Close() error {
+	return f.zipFileCloser.Close()
+}
+
+// openOnly returns a ReadCloser where calling Close will close the zip fs as well.
+func (f *ZipFS) OpenOnly(name string) (io.ReadCloser, error) {
+	r, err := f.Open(name)
+	if err != nil {
+		return nil, err
+	}
+
+	return &wrappedReadCloser{
+		ReadCloser: r,
+		outer:      f,
+	}, nil
+}
+
+type wrappedReadCloser struct {
+	io.ReadCloser
+	outer io.Closer
+}
+
+func (f *wrappedReadCloser) Close() error {
+	_ = f.ReadCloser.Close()
+	return f.outer.Close()
 }