Add exclude file from scan feature (#253)

* Added exclude file from scan feature

* Abort exclusion instead of panicking when pattern isn't valid

* Added UI configuration for exclude patterns

*   * cosmetic fixes
  * changed behavior of exclude function to continue and ignore invalide regex patterns
  * added some more tests (windows networks and continue after regex error)

pkg/api/resolver_mutation_configure.go

@@ -72,6 +72,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		logger.SetLogLevel(input.LogLevel)
 	}
 
+	if input.Excludes != nil {
+		config.Set(config.Exclude, input.Excludes)
+	}
+
 	if err := config.Write(); err != nil {
 		return makeConfigGeneralResult(), err
 	}

pkg/api/resolver_query_configuration.go

@@ -45,6 +45,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		LogOut:                    config.GetLogOut(),
 		LogLevel:                  config.GetLogLevel(),
 		LogAccess:                 config.GetLogAccess(),
+		Excludes:                  config.GetExcludes(),
 	}
 }
 

pkg/manager/config/config.go

@@ -23,6 +23,7 @@ const Password = "password"
 const Database = "database"
 
 const ScrapersPath = "scrapers_path"
+const Exclude = "exclude"
 
 const MaxTranscodeSize = "max_transcode_size"
 const MaxStreamingTranscodeSize = "max_streaming_transcode_size"
@@ -91,6 +92,10 @@ func GetDefaultScrapersPath() string {
 	return fn
 }
 
+func GetExcludes() []string {
+	return viper.GetStringSlice(Exclude)
+}
+
 func GetScrapersPath() string {
 	return viper.GetString(ScrapersPath)
 }

pkg/manager/manager_tasks.go

@@ -2,7 +2,9 @@ package manager
 
 import (
 	"path/filepath"
+	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -78,6 +80,7 @@ func (s *singleton) Scan(useFileMetadata bool) {
 			return
 		}
 
+		results, _ = excludeFiles(results, config.GetExcludes())
 		total := len(results)
 		logger.Infof("Starting scan of %d files. %d New files found", total, s.neededScan(results))
 
@@ -492,3 +495,46 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma
 	}
 	return &totals
 }
+
+func excludeFiles(files []string, patterns []string) ([]string, int) {
+	if patterns == nil {
+		logger.Infof("No excludes in config.")
+		return files, 0
+	} else {
+		var results []string
+		var exclCount int
+		var fileRegexps []*regexp.Regexp
+
+		for _, pattern := range patterns {
+			reg, err := regexp.Compile(strings.ToLower(pattern))
+			if err != nil {
+				logger.Errorf("Exclude :%v", err)
+			} else {
+				fileRegexps = append(fileRegexps, reg)
+			}
+		}
+
+		if len(fileRegexps) == 0 {
+			return files, 0
+		}
+
+		for i := 0; i < len(files); i++ {
+			match := false
+			for _, regPattern := range fileRegexps {
+				if regPattern.Match([]byte(strings.ToLower(files[i]))) {
+					logger.Infof("File %s excluded from scan ", files[i])
+					match = true
+					exclCount++
+					break
+				}
+
+			}
+			//if pattern doesn't match add file to list
+			if !match {
+				results = append(results, files[i])
+			}
+		}
+		logger.Infof("Excluded %d file(s) from scan ", exclCount)
+		return results, exclCount
+	}
+}

pkg/manager/manager_tasks_test.go

@@ -0,0 +1,66 @@
+package manager
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestExcludeFiles(t *testing.T) {
+
+	filenames := []string{
+		"/stash/videos/filename.mp4",
+		"/stash/videos/new filename.mp4",
+		"filename sample.mp4",
+		"/stash/videos/exclude/not wanted.webm",
+		"/stash/videos/exclude/not wanted2.webm",
+		"/somewhere/trash/not wanted.wmv",
+		"/disk2/stash/videos/exclude/!!wanted!!.avi",
+		"/disk2/stash/videos/xcl/not wanted.avi",
+		"/stash/videos/partial.file.001.webm",
+		"/stash/videos/partial.file.002.webm",
+		"/stash/videos/partial.file.003.webm",
+		"/stash/videos/sample file sample.mkv",
+		"/stash/videos/.ckRVp1/.still_encoding.mp4",
+		"c:\\stash\\videos\\exclude\\filename  windows.mp4",
+		"c:\\stash\\videos\\filename  windows.mp4",
+		"\\\\network\\videos\\filename  windows network.mp4",
+		"\\\\network\\share\\windows network wanted.mp4",
+		"\\\\network\\share\\windows network wanted sample.mp4",
+		"\\\\network\\private\\windows.network.skip.mp4"}
+
+	var excludeTests = []struct {
+		testPattern []string
+		expected    int
+	}{
+		{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
+		{[]string{"no_match\\.mp4"}, 0},                                //no match
+		{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3},       //linux
+		{[]string{"/\\.[[:word:]]+/"}, 1},                              //linux hidden dirs (handbrake unraid issue?)
+		{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1},              //windows
+		{[]string{"\\/[/invalid"}, 0},                                  //invalid pattern
+		{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3},        //invalid pattern but continue
+		{[]string{"^\\\\\\\\network"}, 4},                              //windows net share
+		{[]string{"\\\\private\\\\"}, 1},                               //windows net share
+		{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3},               //windows net share
+	}
+	for _, test := range excludeTests {
+		err := runExclude(filenames, test.testPattern, test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+func runExclude(filenames []string, patterns []string, expCount int) error {
+
+	files, count := excludeFiles(filenames, patterns)
+
+	if count != expCount {
+		return fmt.Errorf("Was expecting %d, found %d", expCount, count)
+	}
+	if len(files) != len(filenames)-expCount {
+		return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
+	}
+
+	return nil
+}