Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactor scraping to include related object fields (#6266)

Browse Source 
* Refactor scraper post-processing and process related objects consistently
* Refactor image processing
* Scrape related studio fields consistently
* Don't set image on related objects
This commit is contained in:
WithoutPants
2025-12-02 12:49:44 +11:00
committed by GitHub
parent c6ae43c1d6
commit 84e24eb612
3 changed files with 362 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
pkg/scraper/cache.go
View File

@@ -16,7 +16,6 @@ import (
"github.com/stashapp/stash/pkg/logger"
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/sliceutil"
"github.com/stashapp/stash/pkg/txn"
)
@@ -262,19 +261,23 @@ func (c Cache) ScrapeName(ctx context.Context, id, query string, ty ScrapeConten
return nil, fmt.Errorf("error while name scraping with scraper %s: %w", id, err)
}
ignoredRegex := c.compileExcludeTagPatterns()
var ignoredTags []string
for i, cc := range content {
var thisIgnoredTags []string
content[i], thisIgnoredTags, err = c.postScrape(ctx, cc, ignoredRegex)
if err != nil {
return nil, fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
pp := postScraper{
Cache: c,
excludeTagRE: c.compileExcludeTagPatterns(),
}
if err := c.repository.WithReadTxn(ctx, func(ctx context.Context) error {
for i, cc := range content {
content[i], err = pp.postScrape(ctx, cc)
if err != nil {
return fmt.Errorf("error while post-scraping with scraper %s: %w", id, err)
}
}
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
return nil
}); err != nil {
return nil, err
}
LogIgnoredTags(ignoredTags)
LogIgnoredTags(pp.ignoredTags)
return content, nil
}