Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 12:24:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Move tag exclusion to scrape query resolver (#2391)

Browse Source
This commit is contained in:
WithoutPants
2022-03-20 19:46:12 +11:00
committed by GitHub
parent e4ad42caf0
commit dd0fa48345
4 changed files with 104 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
pkg/scraper/postprocessing.go
View File

@@ -2,8 +2,6 @@ package scraper
import (
"context"
"regexp"
"strings"
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
@@ -50,7 +48,7 @@ func (c Cache) postScrapePerformer(ctx context.Context, p models.ScrapedPerforme
if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
tqb := r.Tag()
tags, err := postProcessTags(c.globalConfig, tqb, p.Tags)
tags, err := postProcessTags(tqb, p.Tags)
if err != nil {
return err
}
@@ -93,7 +91,7 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer
if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
tqb := r.Tag()
tags, err := postProcessTags(c.globalConfig, tqb, p.Tags)
tags, err := postProcessTags(tqb, p.Tags)
if err != nil {
return err
}
@@ -135,7 +133,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene) (
}
}
tags, err := postProcessTags(c.globalConfig, tqb, scene.Tags)
tags, err := postProcessTags(tqb, scene.Tags)
if err != nil {
return err
}
@@ -174,7 +172,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery) (
}
}
tags, err := postProcessTags(c.globalConfig, tqb, g.Tags)
tags, err := postProcessTags(tqb, g.Tags)
if err != nil {
return err
}
@@ -195,31 +193,10 @@ func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery) (
return g, nil
}
func postProcessTags(globalConfig GlobalConfig, tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) {
var ret []*models.ScrapedTag
excludePatterns := globalConfig.GetScraperExcludeTagPatterns()
var excludeRegexps []*regexp.Regexp
for _, excludePattern := range excludePatterns {
reg, err := regexp.Compile(strings.ToLower(excludePattern))
if err != nil {
logger.Errorf("Invalid tag exclusion pattern :%v", err)
} else {
excludeRegexps = append(excludeRegexps, reg)
}
}
var ignoredTags []string
ScrapeTag:
for _, t := range scrapedTags {
for _, reg := range excludeRegexps {
if reg.MatchString(strings.ToLower(t.Name)) {
ignoredTags = append(ignoredTags, t.Name)
continue ScrapeTag
}
}
err := match.ScrapedTag(tqb, t)
if err != nil {
return nil, err
@@ -227,9 +204,5 @@ ScrapeTag:
ret = append(ret, t)
}
if len(ignoredTags) > 0 {
logger.Infof("Scraping ignored tags: %s", strings.Join(ignoredTags, ", "))
}
return ret, nil
}