Add Xpath post processing and performer name query (#333)

* Extend xpath configuration. Support concatenation

* Add parseDate parsing option

* Add regex replacements

* Add xpath query performer by name

* Fix loading spinner on scrape performer

* Change ReplaceAll to Replace
This commit is contained in:
WithoutPants
2020-02-01 09:17:40 +11:00
committed by GitHub
parent 2632f9e971
commit 03c07a429d
4 changed files with 345 additions and 39 deletions

View File

@@ -41,6 +41,9 @@ type scraperTypeConfig struct {
Script []string `yaml:"script,flow"`
Scraper string `yaml:"scraper"`
// for xpath name scraper only
QueryURL string `yaml:"queryURL"`
scraperConfig *scraperConfig
}
@@ -56,6 +59,8 @@ func (c *performerByNameConfig) resolveFn() {
c.performScrape = scrapePerformerNamesScript
} else if c.Action == scraperActionStash {
c.performScrape = scrapePerformerNamesStash
} else if c.Action == scraperActionXPath {
c.performScrape = scrapePerformerNamesXPath
}
}
@@ -266,6 +271,11 @@ func (c scraperConfig) ScrapePerformer(scrapedPerformer models.ScrapedPerformerI
return c.PerformerByFragment.performScrape(c.PerformerByFragment.scraperTypeConfig, scrapedPerformer)
}
// try to match against URL if present
if scrapedPerformer.URL != nil && *scrapedPerformer.URL != "" {
return c.ScrapePerformerURL(*scrapedPerformer.URL)
}
return nil, nil
}