Manager refactor, part 1 (#4298)

* Move BackupDatabase and AnonymiseDatabase to internal/manager
* Rename config.Instance to config.Config
* Rename FFMPEG
* Rework manager and initialization process
* Fix Makefile
* Tweak phasher
* Fix config races
* Fix setup error not clearing

pkg/ffmpeg/downloader.go

@@ -29,7 +29,7 @@ func GetPaths(paths []string) (string, string) {
 
 	// Check if ffmpeg exists in the config directory
 	if ffmpegPath == "" {
-		ffmpegPath = fsutil.FindInPaths(paths, getFFMPEGFilename())
+		ffmpegPath = fsutil.FindInPaths(paths, getFFMpegFilename())
 	}
 	if ffprobePath == "" {
 		ffprobePath = fsutil.FindInPaths(paths, getFFProbeFilename())
@@ -39,7 +39,7 @@ func GetPaths(paths []string) (string, string) {
 }
 
 func Download(ctx context.Context, configDirectory string) error {
-	for _, url := range getFFMPEGURL() {
+	for _, url := range getFFmpegURL() {
 		err := downloadSingle(ctx, configDirectory, url)
 		if err != nil {
 			return err
@@ -47,7 +47,7 @@ func Download(ctx context.Context, configDirectory string) error {
 	}
 
 	// validate that the urls contained what we needed
-	executables := []string{getFFMPEGFilename(), getFFProbeFilename()}
+	executables := []string{getFFMpegFilename(), getFFProbeFilename()}
 	for _, executable := range executables {
 		_, err := os.Stat(filepath.Join(configDirectory, executable))
 		if err != nil {
@@ -173,7 +173,7 @@ func downloadSingle(ctx context.Context, configDirectory, url string) error {
 	return nil
 }
 
-func getFFMPEGURL() []string {
+func getFFmpegURL() []string {
 	var urls []string
 	switch runtime.GOOS {
 	case "darwin":
@@ -195,7 +195,7 @@ func getFFMPEGURL() []string {
 	return urls
 }
 
-func getFFMPEGFilename() string {
+func getFFMpegFilename() string {
 	if runtime.GOOS == "windows" {
 		return "ffmpeg.exe"
 	}
@@ -209,7 +209,7 @@ func getFFProbeFilename() string {
 	return "ffprobe"
 }
 
-// Checks if FFMPEG in the path has the correct flags
+// Checks if ffmpeg in the path has the correct flags
 func pathBinaryHasCorrectFlags() bool {
 	ffmpegPath, err := exec.LookPath("ffmpeg")
 	if err != nil {

pkg/plugin/plugins.go

@@ -15,6 +15,7 @@ import (
 	"path/filepath"
 	"strconv"
 
+	"github.com/stashapp/stash/pkg/fsutil"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/plugin/common"
@@ -123,46 +124,31 @@ func (c *Cache) RegisterSessionStore(sessionStore *session.Store) {
 	c.sessionStore = sessionStore
 }
 
-// LoadPlugins clears the plugin cache and loads from the plugin path.
-// In the event of an error during loading, the cache will be left empty.
-func (c *Cache) LoadPlugins() error {
-	c.plugins = nil
-	plugins, err := loadPlugins(c.config.GetPluginsPath())
-	if err != nil {
-		return err
-	}
-
-	c.plugins = plugins
-	return nil
-}
-
-func loadPlugins(path string) ([]Config, error) {
+// ReloadPlugins clears the plugin cache and loads from the plugin path.
+// If a plugin cannot be loaded, an error is logged and the plugin is skipped.
+func (c *Cache) ReloadPlugins() {
+	path := c.config.GetPluginsPath()
 	plugins := make([]Config, 0)
 
 	logger.Debugf("Reading plugin configs from %s", path)
-	pluginFiles := []string{}
-	err := filepath.Walk(path, func(fp string, f os.FileInfo, err error) error {
+
+	err := fsutil.SymWalk(path, func(fp string, f os.FileInfo, err error) error {
 		if filepath.Ext(fp) == ".yml" {
-			pluginFiles = append(pluginFiles, fp)
+			plugin, err := loadPluginFromYAMLFile(fp)
+			if err != nil {
+				logger.Errorf("Error loading plugin %s: %v", fp, err)
+			} else {
+				plugins = append(plugins, *plugin)
+			}
 		}
 		return nil
 	})
 
 	if err != nil {
-
-		return nil, err
+		logger.Errorf("Error reading plugin configs: %v", err)
 	}
 
-	for _, file := range pluginFiles {
-		plugin, err := loadPluginFromYAMLFile(file)
-		if err != nil {
-			logger.Errorf("Error loading plugin %s: %s", file, err.Error())
-		} else {
-			plugins = append(plugins, *plugin)
-		}
-	}
-
-	return plugins, nil
+	c.plugins = plugins
 }
 
 func (c Cache) enabledPlugins() []Config {

pkg/scraper/cache.go

@@ -133,32 +133,27 @@ func newClient(gc GlobalConfig) *http.Client {
 	return client
 }
 
-// NewCache returns a new Cache loading scraper configurations from the
-// scraper path provided in the global config object. It returns a new
-// instance and an error if the scraper directory could not be loaded.
+// NewCache returns a new Cache.
 //
-// Scraper configurations are loaded from yml files in the provided scrapers
-// directory and any subdirectories.
-func NewCache(globalConfig GlobalConfig, repo Repository) (*Cache, error) {
+// Scraper configurations are loaded from yml files in the scrapers
+// directory in the config and any subdirectories.
+//
+// Does not load scrapers. Scrapers will need to be
+// loaded explicitly using ReloadScrapers.
+func NewCache(globalConfig GlobalConfig, repo Repository) *Cache {
 	// HTTP Client setup
 	client := newClient(globalConfig)
 
-	ret := &Cache{
+	return &Cache{
 		client:       client,
 		globalConfig: globalConfig,
 		repository:   repo,
 	}
-
-	var err error
-	ret.scrapers, err = ret.loadScrapers()
-	if err != nil {
-		return nil, err
-	}
-
-	return ret, nil
 }
 
-func (c *Cache) loadScrapers() (map[string]scraper, error) {
+// ReloadScrapers clears the scraper cache and reloads from the scraper path.
+// If a scraper cannot be loaded, an error is logged and the scraper is skipped.
+func (c *Cache) ReloadScrapers() {
 	path := c.globalConfig.GetScrapersPath()
 	scrapers := make(map[string]scraper)
 
@@ -185,23 +180,9 @@ func (c *Cache) loadScrapers() (map[string]scraper, error) {
 
 	if err != nil {
 		logger.Errorf("Error reading scraper configs: %v", err)
-		return nil, err
-	}
-
-	return scrapers, nil
-}
-
-// ReloadScrapers clears the scraper cache and reloads from the scraper path.
-// In the event of an error during loading, the cache will be left empty.
-func (c *Cache) ReloadScrapers() error {
-	c.scrapers = nil
-	scrapers, err := c.loadScrapers()
-	if err != nil {
-		return err
 	}
 
 	c.scrapers = scrapers
-	return nil
 }
 
 // ListScrapers lists scrapers matching one of the given types.

pkg/sqlite/setup_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stashapp/stash/internal/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/sliceutil"
 	"github.com/stashapp/stash/pkg/sqlite"
@@ -535,6 +536,10 @@ func indexFromID(ids []int, id int) int {
 var db *sqlite.Database
 
 func TestMain(m *testing.M) {
+	// initialise empty config - needed by some migrations
+	_ = config.InitializeEmpty()
+
+
 	ret := runTests(m)
 	os.Exit(ret)
 }