File storage rewrite (#2676)

* Restructure data layer part 2 (#2599)
* Refactor and separate image model
* Refactor image query builder
* Handle relationships in image query builder
* Remove relationship management methods
* Refactor gallery model/query builder
* Add scenes to gallery model
* Convert scene model
* Refactor scene models
* Remove unused methods
* Add unit tests for gallery
* Add image tests
* Add scene tests
* Convert unnecessary scene value pointers to values
* Convert unnecessary pointer values to values
* Refactor scene partial
* Add scene partial tests
* Refactor ImagePartial
* Add image partial tests
* Refactor gallery partial update
* Add partial gallery update tests
* Use zero/null package for null values
* Add files and scan system
* Add sqlite implementation for files/folders
* Add unit tests for files/folders
* Image refactors
* Update image data layer
* Refactor gallery model and creation
* Refactor scene model
* Refactor scenes
* Don't set title from filename
* Allow galleries to freely add/remove images
* Add multiple scene file support to graphql and UI
* Add multiple file support for images in graphql/UI
* Add multiple file for galleries in graphql/UI
* Remove use of some deprecated fields
* Remove scene path usage
* Remove gallery path usage
* Remove path from image
* Move funscript to video file
* Refactor caption detection
* Migrate existing data
* Add post commit/rollback hook system
* Lint. Comment out import/export tests
* Add WithDatabase read only wrapper
* Prepend tasks to list
* Add 32 pre-migration
* Add warnings in release and migration notes

pkg/file/walk.go

@@ -0,0 +1,153 @@
+package file
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// Modified from github.com/facebookgo/symwalk
+
+// BSD License
+
+// For symwalk software
+
+// Copyright (c) 2015, Facebook, Inc. All rights reserved.
+
+// Redistribution and use in source and binary forms, with or without modification,
+// are permitted provided that the following conditions are met:
+
+//  * Redistributions of source code must retain the above copyright notice, this
+//    list of conditions and the following disclaimer.
+
+//  * Redistributions in binary form must reproduce the above copyright notice,
+//    this list of conditions and the following disclaimer in the documentation
+//    and/or other materials provided with the distribution.
+
+//  * Neither the name Facebook nor the names of its contributors may be used to
+//    endorse or promote products derived from this software without specific
+//    prior written permission.
+
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// symwalkFunc calls the provided WalkFn for regular files.
+// However, when it encounters a symbolic link, it resolves the link fully using the
+// filepath.EvalSymlinks function and recursively calls symwalk.Walk on the resolved path.
+// This ensures that unlink filepath.Walk, traversal does not stop at symbolic links.
+//
+// Note that symwalk.Walk does not terminate if there are any non-terminating loops in
+// the file structure.
+func walkSym(f FS, filename string, linkDirname string, walkFn fs.WalkDirFunc) error {
+	symWalkFunc := func(path string, info fs.DirEntry, err error) error {
+
+		if fname, err := filepath.Rel(filename, path); err == nil {
+			path = filepath.Join(linkDirname, fname)
+		} else {
+			return err
+		}
+
+		if err == nil && info.Type()&os.ModeSymlink == os.ModeSymlink {
+			finalPath, err := filepath.EvalSymlinks(path)
+			if err != nil {
+				// don't bail out if symlink is invalid
+				return walkFn(path, info, err)
+			}
+			info, err := f.Lstat(finalPath)
+			if err != nil {
+				return walkFn(path, &statDirEntry{
+					info: info,
+				}, err)
+			}
+			if info.IsDir() {
+				return walkSym(f, finalPath, path, walkFn)
+			}
+		}
+
+		return walkFn(path, info, err)
+	}
+	return fsWalk(f, filename, symWalkFunc)
+}
+
+// symWalk extends filepath.Walk to also follow symlinks
+func symWalk(fs FS, path string, walkFn fs.WalkDirFunc) error {
+	return walkSym(fs, path, path, walkFn)
+}
+
+type statDirEntry struct {
+	info fs.FileInfo
+}
+
+func (d *statDirEntry) Name() string               { return d.info.Name() }
+func (d *statDirEntry) IsDir() bool                { return d.info.IsDir() }
+func (d *statDirEntry) Type() fs.FileMode          { return d.info.Mode().Type() }
+func (d *statDirEntry) Info() (fs.FileInfo, error) { return d.info, nil }
+
+func fsWalk(f FS, root string, fn fs.WalkDirFunc) error {
+	info, err := f.Lstat(root)
+	if err != nil {
+		err = fn(root, nil, err)
+	} else {
+		err = walkDir(f, root, &statDirEntry{info}, fn)
+	}
+	if errors.Is(err, fs.SkipDir) {
+		return nil
+	}
+	return err
+}
+
+func walkDir(f FS, path string, d fs.DirEntry, walkDirFn fs.WalkDirFunc) error {
+	if err := walkDirFn(path, d, nil); err != nil || !d.IsDir() {
+		if errors.Is(err, fs.SkipDir) && d.IsDir() {
+			// Successfully skipped directory.
+			err = nil
+		}
+		return err
+	}
+
+	dirs, err := readDir(f, path)
+	if err != nil {
+		// Second call, to report ReadDir error.
+		err = walkDirFn(path, d, err)
+		if err != nil {
+			return err
+		}
+	}
+
+	for _, d1 := range dirs {
+		path1 := filepath.Join(path, d1.Name())
+		if err := walkDir(f, path1, d1, walkDirFn); err != nil {
+			if errors.Is(err, fs.SkipDir) {
+				break
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+// readDir reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func readDir(fs FS, dirname string) ([]fs.DirEntry, error) {
+	f, err := fs.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	dirs, err := f.ReadDir(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+	sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
+	return dirs, nil
+}