Added exclude patterns support for Clean Task (#274)

* Added exclude patterns support for Clean Task

* Added test file

* Refactoring and cosmetic fixes

* * Replace Match with MatchString

pkg/manager/exclude_files.go

@@ -0,0 +1,89 @@
+package manager
+
+import (
+	"github.com/stashapp/stash/pkg/logger"
+	"regexp"
+	"strings"
+)
+
+func excludeFiles(files []string, patterns []string) ([]string, int) {
+	if patterns == nil {
+		logger.Infof("No exclude patterns in config")
+		return files, 0
+	} else {
+		var results []string
+		var exclCount int
+
+		fileRegexps := generateRegexps(patterns)
+
+		if len(fileRegexps) == 0 {
+			logger.Infof("Excluded 0 files from scan")
+			return files, 0
+		}
+
+		for i := 0; i < len(files); i++ {
+			if matchFileSimple(files[i], fileRegexps) {
+				logger.Infof("File matched pattern. Excluding:\"%s\"", files[i])
+				exclCount++
+			} else {
+
+				//if pattern doesn't match add file to list
+				results = append(results, files[i])
+			}
+		}
+		logger.Infof("Excluded %d file(s) from scan", exclCount)
+		return results, exclCount
+	}
+}
+
+func matchFile(file string, patterns []string) bool {
+	if patterns == nil {
+		logger.Infof("No exclude patterns in config.")
+
+	} else {
+		fileRegexps := generateRegexps(patterns)
+
+		if len(fileRegexps) == 0 {
+			return false
+		}
+
+		for _, regPattern := range fileRegexps {
+			if regPattern.MatchString(strings.ToLower(file)) {
+				return true
+			}
+
+		}
+	}
+
+	return false
+}
+
+func generateRegexps(patterns []string) []*regexp.Regexp {
+
+	var fileRegexps []*regexp.Regexp
+
+	for _, pattern := range patterns {
+		reg, err := regexp.Compile(strings.ToLower(pattern))
+		if err != nil {
+			logger.Errorf("Exclude :%v", err)
+		} else {
+			fileRegexps = append(fileRegexps, reg)
+		}
+	}
+
+	if len(fileRegexps) == 0 {
+		return nil
+	} else {
+		return fileRegexps
+	}
+
+}
+
+func matchFileSimple(file string, regExps []*regexp.Regexp) bool {
+	for _, regPattern := range regExps {
+		if regPattern.MatchString(strings.ToLower(file)) {
+			return true
+		}
+	}
+	return false
+}

pkg/manager/exclude_files_test.go

@@ -0,0 +1,91 @@
+package manager
+
+import (
+	"fmt"
+	"github.com/stashapp/stash/pkg/logger"
+	"testing"
+)
+
+var excludeTestFilenames = []string{
+	"/stash/videos/filename.mp4",
+	"/stash/videos/new filename.mp4",
+	"filename sample.mp4",
+	"/stash/videos/exclude/not wanted.webm",
+	"/stash/videos/exclude/not wanted2.webm",
+	"/somewhere/trash/not wanted.wmv",
+	"/disk2/stash/videos/exclude/!!wanted!!.avi",
+	"/disk2/stash/videos/xcl/not wanted.avi",
+	"/stash/videos/partial.file.001.webm",
+	"/stash/videos/partial.file.002.webm",
+	"/stash/videos/partial.file.003.webm",
+	"/stash/videos/sample file sample.mkv",
+	"/stash/videos/.ckRVp1/.still_encoding.mp4",
+	"c:\\stash\\videos\\exclude\\filename  windows.mp4",
+	"c:\\stash\\videos\\filename  windows.mp4",
+	"\\\\network\\videos\\filename  windows network.mp4",
+	"\\\\network\\share\\windows network wanted.mp4",
+	"\\\\network\\share\\windows network wanted sample.mp4",
+	"\\\\network\\private\\windows.network.skip.mp4"}
+
+var excludeTests = []struct {
+	testPattern []string
+	expected    int
+}{
+	{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
+	{[]string{"no_match\\.mp4"}, 0},                                //no match
+	{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3},       //linux
+	{[]string{"/\\.[[:word:]]+/"}, 1},                              //linux hidden dirs (handbrake unraid issue?)
+	{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1},              //windows
+	{[]string{"\\/[/invalid"}, 0},                                  //invalid pattern
+	{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3},        //invalid pattern but continue
+	{[]string{"^\\\\\\\\network"}, 4},                              //windows net share
+	{[]string{"\\\\private\\\\"}, 1},                               //windows net share
+	{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3},               //windows net share
+}
+
+func TestExcludeFiles(t *testing.T) {
+	for _, test := range excludeTests {
+		err := runExclude(excludeTestFilenames, test.testPattern, test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+func runExclude(filenames []string, patterns []string, expCount int) error {
+
+	files, count := excludeFiles(filenames, patterns)
+
+	if count != expCount {
+		return fmt.Errorf("Was expecting %d, found %d", expCount, count)
+	}
+	if len(files) != len(filenames)-expCount {
+		return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
+	}
+
+	return nil
+}
+
+func TestMatchFile(t *testing.T) {
+	for _, test := range excludeTests {
+		err := runMatch(excludeTestFilenames, test.testPattern, test.expected)
+		if err != nil {
+			t.Error(err)
+		}
+	}
+}
+
+func runMatch(filenames []string, patterns []string, expCount int) error {
+	count := 0
+	for _, file := range filenames {
+		if matchFile(file, patterns) {
+			logger.Infof("File \"%s\" matched pattern\n", file)
+			count++
+		}
+	}
+	if count != expCount {
+		return fmt.Errorf("Was expecting %d, found %d", expCount, count)
+	}
+
+	return nil
+}

pkg/manager/manager_tasks.go

@@ -1,18 +1,15 @@
 package manager
 
 import (
-	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
 	"github.com/bmatcuk/doublestar"
 	"github.com/stashapp/stash/pkg/logger"
 	"github.com/stashapp/stash/pkg/manager/config"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/utils"
+	"path/filepath"
+	"strconv"
+	"sync"
+	"time"
 )
 
 type TaskStatus struct {
@@ -414,7 +411,7 @@ func (s *singleton) Clean() {
 			}
 
 			if scene == nil {
-				logger.Errorf("nil scene, skipping generate")
+				logger.Errorf("nil scene, skipping Clean")
 				continue
 			}
 
@@ -495,46 +492,3 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma
 	}
 	return &totals
 }
-
-func excludeFiles(files []string, patterns []string) ([]string, int) {
-	if patterns == nil {
-		logger.Infof("No excludes in config.")
-		return files, 0
-	} else {
-		var results []string
-		var exclCount int
-		var fileRegexps []*regexp.Regexp
-
-		for _, pattern := range patterns {
-			reg, err := regexp.Compile(strings.ToLower(pattern))
-			if err != nil {
-				logger.Errorf("Exclude :%v", err)
-			} else {
-				fileRegexps = append(fileRegexps, reg)
-			}
-		}
-
-		if len(fileRegexps) == 0 {
-			return files, 0
-		}
-
-		for i := 0; i < len(files); i++ {
-			match := false
-			for _, regPattern := range fileRegexps {
-				if regPattern.Match([]byte(strings.ToLower(files[i]))) {
-					logger.Infof("File %s excluded from scan ", files[i])
-					match = true
-					exclCount++
-					break
-				}
-
-			}
-			//if pattern doesn't match add file to list
-			if !match {
-				results = append(results, files[i])
-			}
-		}
-		logger.Infof("Excluded %d file(s) from scan ", exclCount)
-		return results, exclCount
-	}
-}

pkg/manager/manager_tasks_test.go

@@ -1,66 +0,0 @@
-package manager
-
-import (
-	"fmt"
-	"testing"
-)
-
-func TestExcludeFiles(t *testing.T) {
-
-	filenames := []string{
-		"/stash/videos/filename.mp4",
-		"/stash/videos/new filename.mp4",
-		"filename sample.mp4",
-		"/stash/videos/exclude/not wanted.webm",
-		"/stash/videos/exclude/not wanted2.webm",
-		"/somewhere/trash/not wanted.wmv",
-		"/disk2/stash/videos/exclude/!!wanted!!.avi",
-		"/disk2/stash/videos/xcl/not wanted.avi",
-		"/stash/videos/partial.file.001.webm",
-		"/stash/videos/partial.file.002.webm",
-		"/stash/videos/partial.file.003.webm",
-		"/stash/videos/sample file sample.mkv",
-		"/stash/videos/.ckRVp1/.still_encoding.mp4",
-		"c:\\stash\\videos\\exclude\\filename  windows.mp4",
-		"c:\\stash\\videos\\filename  windows.mp4",
-		"\\\\network\\videos\\filename  windows network.mp4",
-		"\\\\network\\share\\windows network wanted.mp4",
-		"\\\\network\\share\\windows network wanted sample.mp4",
-		"\\\\network\\private\\windows.network.skip.mp4"}
-
-	var excludeTests = []struct {
-		testPattern []string
-		expected    int
-	}{
-		{[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic
-		{[]string{"no_match\\.mp4"}, 0},                                //no match
-		{[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3},       //linux
-		{[]string{"/\\.[[:word:]]+/"}, 1},                              //linux hidden dirs (handbrake unraid issue?)
-		{[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1},              //windows
-		{[]string{"\\/[/invalid"}, 0},                                  //invalid pattern
-		{[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3},        //invalid pattern but continue
-		{[]string{"^\\\\\\\\network"}, 4},                              //windows net share
-		{[]string{"\\\\private\\\\"}, 1},                               //windows net share
-		{[]string{"\\\\private\\\\", "sample\\.mp4"}, 3},               //windows net share
-	}
-	for _, test := range excludeTests {
-		err := runExclude(filenames, test.testPattern, test.expected)
-		if err != nil {
-			t.Error(err)
-		}
-	}
-}
-
-func runExclude(filenames []string, patterns []string, expCount int) error {
-
-	files, count := excludeFiles(filenames, patterns)
-
-	if count != expCount {
-		return fmt.Errorf("Was expecting %d, found %d", expCount, count)
-	}
-	if len(files) != len(filenames)-expCount {
-		return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files))
-	}
-
-	return nil
-}

pkg/manager/task_clean.go

@@ -21,8 +21,12 @@ func (t *CleanTask) Start(wg *sync.WaitGroup) {
 
 	if t.fileExists(t.Scene.Path) && t.pathInStash() {
 		logger.Debugf("File Found: %s", t.Scene.Path)
+		if matchFile(t.Scene.Path, config.GetExcludes()) {
+			logger.Infof("File matched regex. Cleaning: \"%s\"", t.Scene.Path)
+			t.deleteScene(t.Scene.ID)
+		}
 	} else {
-		logger.Infof("File not found. Cleaning: %s", t.Scene.Path)
+		logger.Infof("File not found. Cleaning: \"%s\"", t.Scene.Path)
 		t.deleteScene(t.Scene.ID)
 	}
 }

ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx

@@ -1,4 +1,5 @@
 import {
+  AnchorButton,
   Button,
   Divider,
   FormGroup,
@@ -177,7 +178,6 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
 
         <FormGroup
           label="Excluded Patterns"
-          helperText="Regexps of files/paths to exclude from Scan"
         >
 
        { (excludes) ? excludes.map((regexp, i) => {
@@ -192,6 +192,17 @@ export const SettingsConfigurationPanel: FunctionComponent<IProps> = (props: IPr
        }
 
           <Button icon="plus" minimal={true} onClick={(e: any) => excludeAddRegex()} />
+          <div>
+            <p>
+              <AnchorButton
+                href="https://github.com/stashapp/stash/wiki/Exclude-file-configuration"
+                rightIcon="help"
+                text="Regexps of files/paths to exclude from Scan and add to Clean"
+                minimal={true}
+                target="_blank" 
+              />
+            </p>
+          </div>
         </FormGroup>
       </FormGroup>