Add exclude file from scan feature (#253)

* Added exclude file from scan feature

* Abort exclusion instead of panicking when pattern isn't valid

* Added UI configuration for exclude patterns

*   * cosmetic fixes
  * changed behavior of exclude function to continue and ignore invalide regex patterns
  * added some more tests (windows networks and continue after regex error)

graphql/documents/data/config.graphql

@@ -10,6 +10,7 @@ fragment ConfigGeneralData on ConfigGeneralResult {
   logOut
   logLevel
   logAccess
+  excludes
 }
 
 fragment ConfigInterfaceData on ConfigInterfaceResult {
@@ -29,4 +30,4 @@ fragment ConfigData on ConfigResult {
   interface {
     ...ConfigInterfaceData
   }
-}
+}

graphql/schema/types/config.graphql

@@ -30,6 +30,8 @@ input ConfigGeneralInput {
   logLevel: String!
   """Whether to log http access"""
   logAccess: Boolean!
+  """Array of file regexp to exclude from Scan"""
+  excludes: [String!]
 }
 
 type ConfigGeneralResult {
@@ -55,6 +57,8 @@ type ConfigGeneralResult {
   logLevel: String!
   """Whether to log http access"""
   logAccess: Boolean!
+  """Array of file regexp to exclude from Scan"""
+  excludes: [String!]!
 }
 
 input ConfigInterfaceInput {
@@ -93,4 +97,4 @@ type ConfigInterfaceResult {
 type ConfigResult {
   general: ConfigGeneralResult!
   interface: ConfigInterfaceResult!
-}
+}

pkg/api/resolver_mutation_configure.go

@@ -72,6 +72,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
 		logger.SetLogLevel(input.LogLevel)
 	}
 
+	if input.Excludes != nil {
+		config.Set(config.Exclude, input.Excludes)
+	}
+
 	if err := config.Write(); err != nil {
 		return makeConfigGeneralResult(), err
 	}

pkg/api/resolver_query_configuration.go

@@ -45,6 +45,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult {
 		LogOut:                    config.GetLogOut(),
 		LogLevel:                  config.GetLogLevel(),
 		LogAccess:                 config.GetLogAccess(),
+		Excludes:                  config.GetExcludes(),
 	}
 }
 

pkg/manager/config/config.go

@@ -23,6 +23,7 @@ const Password = "password"
 const Database = "database"
 
 const ScrapersPath = "scrapers_path"
+const Exclude = "exclude"
 
 const MaxTranscodeSize = "max_transcode_size"
 const MaxStreamingTranscodeSize = "max_streaming_transcode_size"
@@ -91,6 +92,10 @@ func GetDefaultScrapersPath() string {
 	return fn
 }
 
+func GetExcludes() []string {
+	return viper.GetStringSlice(Exclude)
+}
+
 func GetScrapersPath() string {
 	return viper.GetString(ScrapersPath)
 }

pkg/manager/manager_tasks.go

@@ -2,7 +2,9 @@ package manager
 
 import (
 	"path/filepath"
+	"regexp"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -78,6 +80,7 @@ func (s *singleton) Scan(useFileMetadata bool) {
 			return
 		}
 
+		results, _ = excludeFiles(results, config.GetExcludes())
 		total := len(results)
 		logger.Infof("Starting scan of %d files. %d New files found", total, s.neededScan(results))
 
@@ -492,3 +495,46 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma
 	}
 	return &totals
 }
+
+func excludeFiles(files []string, patterns []string) ([]string, int) {
+	if patterns == nil {
+		logger.Infof("No excludes in config.")
+		return files, 0
+	} else {
+		var results []string
+		var exclCount int
+		var fileRegexps []*regexp.Regexp
+
+		for _, pattern := range patterns {
+			reg, err := regexp.Compile(strings.ToLower(pattern))
+			if err != nil {
+				logger.Errorf("Exclude :%v", err)
+			} else {
+				fileRegexps = append(fileRegexps, reg)
+			}
+		}
+
+		if len(fileRegexps) == 0 {
+			return files, 0
+		}
+
+		for i := 0; i < len(files); i++ {
+			match := false
+			for _, regPattern := range fileRegexps {
+				if regPattern.Match([]byte(strings.ToLower(files[i]))) {
+					logger.Infof("File %s excluded from scan ", files[i])
+					match = true
+					exclCount++
+					break
+				}
+
+			}
+			//if pattern doesn't match add file to list
+			if !match {
+				results = append(results, files[i])
+			}
+		}
+		logger.Infof("Excluded %d file(s) from scan ", exclCount)
+		return results, exclCount
+	}
+}

pkg/manager/manager_tasks_test.go

@@ -0,0 +1,66 @@
+package manager
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestExcludeFiles(t *testing.T) {
+
+	filenames := []string{
+		"/stash/videos/filename.mp4",
+		"/stash/videos/new filename.mp4",
+		"filename sample.mp4",
+		"/stash/videos/exclude/not wanted.webm",
+		"/stash/videos/exclude/not wanted2.webm",
+		"/somewhere/trash/not wanted.wmv",
+		"/disk2/stash/videos/exclude/!!wanted!!.avi",
+		"/disk2/stash/videos/xcl/not wanted.avi",
+		"/stash/videos/partial.file.001.webm",
+		"/stash/videos/partial.file.002.webm",
+		"/stash/videos/partial.file.003.webm",
+		"/stash/videos/sample file sample.mkv",
+		"/stash/videos/.ckRVp1/.still_encoding.mp4",
+		"c:\\stash\\videos\\exclude\\filename  windows.mp4",
+		"c:\\stash\\videos\\filename  windows.mp4",
+		"\\\\network\\videos\\filename  windows network.mp4",
+		"\\\\network\\share\\windows network wanted.mp4",
+		"\\\\network\\share\\windows network wanted sample.mp4",
+		"\\\\network\\private\\windows.network.skip.mp4"}
+
+	var excludeTests = []struct {
+		testPattern []string
+		expected    int
+	}{
+		{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
+		{[]string{"no_match\\.mp4"}, 0},                                //no match
+		{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3},       //linux
+		{[]string{"/\\.[[:word:]]+/"}, 1},                              //linux hidden dirs (handbrake unraid issue?)
+		{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1},              //windows
+		{[]string{"\\/[/invalid"}, 0},                                  //invalid pattern
+		{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3},        //invalid pattern but continue
+		{[]string{"^\\\\\\\\network"}, 4},                              //windows net share
+		{[]string{"\\\\private\\\\"}, 1},                               //windows net share
+		{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3},               //windows net share
+	}
+	for _, test := range excludeTests {
+		err := runExclude(filenames, test.testPattern, test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+func runExclude(filenames []string, patterns []string, expCount int) error {
+
+	files, count := excludeFiles(filenames, patterns)
+
+	if count != expCount {
+		return fmt.Errorf("Was expecting %d, found %d", expCount, count)
+	}
+	if len(files) != len(filenames)-expCount {
+		return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
+	}
+
+	return nil
+}

ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx

@@ -33,6 +33,7 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
   const [logOut, setLogOut] = useState<boolean>(true);
   const [logLevel, setLogLevel] = useState<string>("Info");
   const [logAccess, setLogAccess] = useState<boolean>(true);
+  const [excludes, setExcludes] = useState<(string)[]>([]);
 
   const { data, error, loading } = StashService.useConfiguration();
 
@@ -48,6 +49,8 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
     logOut,
     logLevel,
     logAccess,
+    excludes,
+
   });
 
   useEffect(() => {
@@ -65,6 +68,7 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
       setLogOut(conf.general.logOut);
       setLogLevel(conf.general.logLevel);
       setLogAccess(conf.general.logAccess);
+      setExcludes(conf.general.excludes);
     }
   }, [data]);
 
@@ -72,6 +76,28 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
     setStashes(directories);
   }
 
+  function excludeRegexChanged(idx: number, value: string) {
+    const newExcludes = excludes.map((regex, i)=> {
+      const ret = ( idx !== i ) ? regex : value ;
+      return ret
+      })
+    setExcludes(newExcludes);
+  }
+
+  function excludeRemoveRegex(idx: number) {
+    const newExcludes = excludes.filter((regex, i) => i!== idx );
+
+    setExcludes(newExcludes);
+  }
+
+  function excludeAddRegex() {
+    const demo = "sample\\.mp4$"
+    const newExcludes = excludes.concat(demo);
+
+    setExcludes(newExcludes);
+  }
+
+
   async function onSave() {
     try {
       const result = await updateGeneralConfig();
@@ -148,6 +174,25 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
         >
           <InputGroup value={generatedPath} onChange={(e: any) => setGeneratedPath(e.target.value)} />
         </FormGroup>
+
+        <FormGroup
+          label="Excluded Patterns"
+          helperText="Regexps of files/paths to exclude from Scan"
+        >
+
+       { (excludes) ? excludes.map((regexp, i) => {
+         return(
+           <InputGroup
+             value={regexp}
+             onChange={(e: any) => excludeRegexChanged(i, e.target.value)}
+             rightElement={<Button icon="minus" minimal={true} intent="danger" onClick={(e: any) => excludeRemoveRegex(i)} />}
+           />
+           );
+         }) : null
+       }
+
+          <Button icon="plus" minimal={true} onClick={(e: any) => excludeAddRegex()} />
+        </FormGroup>
       </FormGroup>
       
       <Divider />