From 0714cbfa34e24be87d1c805e34e3d52df3aeb559 Mon Sep 17 00:00:00 2001 From: bnkai <48220860+bnkai@users.noreply.github.com> Date: Tue, 17 Dec 2019 16:26:16 +0200 Subject: [PATCH] Add exclude file from scan feature (#253) * Added exclude file from scan feature * Abort exclusion instead of panicking when pattern isn't valid * Added UI configuration for exclude patterns * * cosmetic fixes * changed behavior of exclude function to continue and ignore invalide regex patterns * added some more tests (windows networks and continue after regex error) --- graphql/documents/data/config.graphql | 3 +- graphql/schema/types/config.graphql | 6 +- pkg/api/resolver_mutation_configure.go | 4 ++ pkg/api/resolver_query_configuration.go | 1 + pkg/manager/config/config.go | 5 ++ pkg/manager/manager_tasks.go | 46 +++++++++++++ pkg/manager/manager_tasks_test.go | 66 +++++++++++++++++++ .../Settings/SettingsConfigurationPanel.tsx | 45 +++++++++++++ 8 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 pkg/manager/manager_tasks_test.go diff --git a/graphql/documents/data/config.graphql b/graphql/documents/data/config.graphql index a62ca9c24..57cd64c94 100644 --- a/graphql/documents/data/config.graphql +++ b/graphql/documents/data/config.graphql @@ -10,6 +10,7 @@ fragment ConfigGeneralData on ConfigGeneralResult { logOut logLevel logAccess + excludes } fragment ConfigInterfaceData on ConfigInterfaceResult { @@ -29,4 +30,4 @@ fragment ConfigData on ConfigResult { interface { ...ConfigInterfaceData } -} \ No newline at end of file +} diff --git a/graphql/schema/types/config.graphql b/graphql/schema/types/config.graphql index 937b60855..2d90c24f4 100644 --- a/graphql/schema/types/config.graphql +++ b/graphql/schema/types/config.graphql @@ -30,6 +30,8 @@ input ConfigGeneralInput { logLevel: String! """Whether to log http access""" logAccess: Boolean! + """Array of file regexp to exclude from Scan""" + excludes: [String!] } type ConfigGeneralResult { @@ -55,6 +57,8 @@ type ConfigGeneralResult { logLevel: String! """Whether to log http access""" logAccess: Boolean! + """Array of file regexp to exclude from Scan""" + excludes: [String!]! } input ConfigInterfaceInput { @@ -93,4 +97,4 @@ type ConfigInterfaceResult { type ConfigResult { general: ConfigGeneralResult! interface: ConfigInterfaceResult! -} \ No newline at end of file +} diff --git a/pkg/api/resolver_mutation_configure.go b/pkg/api/resolver_mutation_configure.go index e7da15f3b..2b9273caf 100644 --- a/pkg/api/resolver_mutation_configure.go +++ b/pkg/api/resolver_mutation_configure.go @@ -72,6 +72,10 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co logger.SetLogLevel(input.LogLevel) } + if input.Excludes != nil { + config.Set(config.Exclude, input.Excludes) + } + if err := config.Write(); err != nil { return makeConfigGeneralResult(), err } diff --git a/pkg/api/resolver_query_configuration.go b/pkg/api/resolver_query_configuration.go index 18b227748..80ddf8bd2 100644 --- a/pkg/api/resolver_query_configuration.go +++ b/pkg/api/resolver_query_configuration.go @@ -45,6 +45,7 @@ func makeConfigGeneralResult() *models.ConfigGeneralResult { LogOut: config.GetLogOut(), LogLevel: config.GetLogLevel(), LogAccess: config.GetLogAccess(), + Excludes: config.GetExcludes(), } } diff --git a/pkg/manager/config/config.go b/pkg/manager/config/config.go index 9005e55ac..2b42c40fc 100644 --- a/pkg/manager/config/config.go +++ b/pkg/manager/config/config.go @@ -23,6 +23,7 @@ const Password = "password" const Database = "database" const ScrapersPath = "scrapers_path" +const Exclude = "exclude" const MaxTranscodeSize = "max_transcode_size" const MaxStreamingTranscodeSize = "max_streaming_transcode_size" @@ -91,6 +92,10 @@ func GetDefaultScrapersPath() string { return fn } +func GetExcludes() []string { + return viper.GetStringSlice(Exclude) +} + func GetScrapersPath() string { return viper.GetString(ScrapersPath) } diff --git a/pkg/manager/manager_tasks.go b/pkg/manager/manager_tasks.go index c8e729cda..8668b2b69 100644 --- a/pkg/manager/manager_tasks.go +++ b/pkg/manager/manager_tasks.go @@ -2,7 +2,9 @@ package manager import ( "path/filepath" + "regexp" "strconv" + "strings" "sync" "time" @@ -78,6 +80,7 @@ func (s *singleton) Scan(useFileMetadata bool) { return } + results, _ = excludeFiles(results, config.GetExcludes()) total := len(results) logger.Infof("Starting scan of %d files. %d New files found", total, s.neededScan(results)) @@ -492,3 +495,46 @@ func (s *singleton) neededGenerate(scenes []*models.Scene, sprites, previews, ma } return &totals } + +func excludeFiles(files []string, patterns []string) ([]string, int) { + if patterns == nil { + logger.Infof("No excludes in config.") + return files, 0 + } else { + var results []string + var exclCount int + var fileRegexps []*regexp.Regexp + + for _, pattern := range patterns { + reg, err := regexp.Compile(strings.ToLower(pattern)) + if err != nil { + logger.Errorf("Exclude :%v", err) + } else { + fileRegexps = append(fileRegexps, reg) + } + } + + if len(fileRegexps) == 0 { + return files, 0 + } + + for i := 0; i < len(files); i++ { + match := false + for _, regPattern := range fileRegexps { + if regPattern.Match([]byte(strings.ToLower(files[i]))) { + logger.Infof("File %s excluded from scan ", files[i]) + match = true + exclCount++ + break + } + + } + //if pattern doesn't match add file to list + if !match { + results = append(results, files[i]) + } + } + logger.Infof("Excluded %d file(s) from scan ", exclCount) + return results, exclCount + } +} diff --git a/pkg/manager/manager_tasks_test.go b/pkg/manager/manager_tasks_test.go new file mode 100644 index 000000000..93d2a07eb --- /dev/null +++ b/pkg/manager/manager_tasks_test.go @@ -0,0 +1,66 @@ +package manager + +import ( + "fmt" + "testing" +) + +func TestExcludeFiles(t *testing.T) { + + filenames := []string{ + "/stash/videos/filename.mp4", + "/stash/videos/new filename.mp4", + "filename sample.mp4", + "/stash/videos/exclude/not wanted.webm", + "/stash/videos/exclude/not wanted2.webm", + "/somewhere/trash/not wanted.wmv", + "/disk2/stash/videos/exclude/!!wanted!!.avi", + "/disk2/stash/videos/xcl/not wanted.avi", + "/stash/videos/partial.file.001.webm", + "/stash/videos/partial.file.002.webm", + "/stash/videos/partial.file.003.webm", + "/stash/videos/sample file sample.mkv", + "/stash/videos/.ckRVp1/.still_encoding.mp4", + "c:\\stash\\videos\\exclude\\filename windows.mp4", + "c:\\stash\\videos\\filename windows.mp4", + "\\\\network\\videos\\filename windows network.mp4", + "\\\\network\\share\\windows network wanted.mp4", + "\\\\network\\share\\windows network wanted sample.mp4", + "\\\\network\\private\\windows.network.skip.mp4"} + + var excludeTests = []struct { + testPattern []string + expected int + }{ + {[]string{"sample\\.mp4$", "trash", "\\.[\\d]{3}\\.webm$"}, 6}, //generic + {[]string{"no_match\\.mp4"}, 0}, //no match + {[]string{"^/stash/videos/exclude/", "/videos/xcl/"}, 3}, //linux + {[]string{"/\\.[[:word:]]+/"}, 1}, //linux hidden dirs (handbrake unraid issue?) + {[]string{"c:\\\\stash\\\\videos\\\\exclude"}, 1}, //windows + {[]string{"\\/[/invalid"}, 0}, //invalid pattern + {[]string{"\\/[/invalid", "sample\\.[[:alnum:]]+$"}, 3}, //invalid pattern but continue + {[]string{"^\\\\\\\\network"}, 4}, //windows net share + {[]string{"\\\\private\\\\"}, 1}, //windows net share + {[]string{"\\\\private\\\\", "sample\\.mp4"}, 3}, //windows net share + } + for _, test := range excludeTests { + err := runExclude(filenames, test.testPattern, test.expected) + if err != nil { + t.Error(err) + } + } +} + +func runExclude(filenames []string, patterns []string, expCount int) error { + + files, count := excludeFiles(filenames, patterns) + + if count != expCount { + return fmt.Errorf("Was expecting %d, found %d", expCount, count) + } + if len(files) != len(filenames)-expCount { + return fmt.Errorf("Returned list should have %d files, not %d ", len(filenames)-expCount, len(files)) + } + + return nil +} diff --git a/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx b/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx index 268830054..300ec7f88 100644 --- a/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx +++ b/ui/v2/src/components/Settings/SettingsConfigurationPanel.tsx @@ -33,6 +33,7 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr const [logOut, setLogOut] = useState(true); const [logLevel, setLogLevel] = useState("Info"); const [logAccess, setLogAccess] = useState(true); + const [excludes, setExcludes] = useState<(string)[]>([]); const { data, error, loading } = StashService.useConfiguration(); @@ -48,6 +49,8 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr logOut, logLevel, logAccess, + excludes, + }); useEffect(() => { @@ -65,6 +68,7 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr setLogOut(conf.general.logOut); setLogLevel(conf.general.logLevel); setLogAccess(conf.general.logAccess); + setExcludes(conf.general.excludes); } }, [data]); @@ -72,6 +76,28 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr setStashes(directories); } + function excludeRegexChanged(idx: number, value: string) { + const newExcludes = excludes.map((regex, i)=> { + const ret = ( idx !== i ) ? regex : value ; + return ret + }) + setExcludes(newExcludes); + } + + function excludeRemoveRegex(idx: number) { + const newExcludes = excludes.filter((regex, i) => i!== idx ); + + setExcludes(newExcludes); + } + + function excludeAddRegex() { + const demo = "sample\\.mp4$" + const newExcludes = excludes.concat(demo); + + setExcludes(newExcludes); + } + + async function onSave() { try { const result = await updateGeneralConfig(); @@ -148,6 +174,25 @@ export const SettingsConfigurationPanel: FunctionComponent = (props: IPr > setGeneratedPath(e.target.value)} /> + + + + { (excludes) ? excludes.map((regexp, i) => { + return( + excludeRegexChanged(i, e.target.value)} + rightElement={