Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Xpath scraping from URL (#285)

Browse Source 
* Add xpath performer and scene scraping

* Add studio scraping

* Refactor code

* Fix compile error

* Don't overwrite performer URL during a scrape
This commit is contained in:
WithoutPants
2020-01-05 03:39:33 +11:00
committed by Leopere
parent d35f3a9b10
commit 7fdaccf669
93 changed files with 174400 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
pkg/scraper/config.go
View File

@@ -19,24 +19,27 @@ type scraperAction string
const (
scraperActionScript scraperAction = "script"
scraperActionStash scraperAction = "stash"
scraperActionXPath scraperAction = "scrapeXPath"
)
var allScraperAction = []scraperAction{
scraperActionScript,
scraperActionStash,
scraperActionXPath,
}
func (e scraperAction) IsValid() bool {
switch e {
case scraperActionScript:
case scraperActionScript, scraperActionStash, scraperActionXPath:
return true
}
return false
}
type scraperTypeConfig struct {
Action scraperAction `yaml:"action"`
Script []string `yaml:"script,flow"`
Action scraperAction `yaml:"action"`
Script []string `yaml:"script,flow"`
Scraper string `yaml:"scraper"`
scraperConfig *scraperConfig
}
@@ -96,6 +99,8 @@ type scrapePerformerByURLConfig struct {
func (c *scrapePerformerByURLConfig) resolveFn() {
if c.Action == scraperActionScript {
c.performScrape = scrapePerformerURLScript
} else if c.Action == scraperActionXPath {
c.performScrape = scrapePerformerURLXpath
}
}
@@ -124,6 +129,8 @@ type scrapeSceneByURLConfig struct {
func (c *scrapeSceneByURLConfig) resolveFn() {
if c.Action == scraperActionScript {
c.performScrape = scrapeSceneURLScript
} else if c.Action == scraperActionXPath {
c.performScrape = scrapeSceneURLXPath
}
}
@@ -135,7 +142,9 @@ type scraperConfig struct {
PerformerByURL []*scrapePerformerByURLConfig `yaml:"performerByURL"`
SceneByFragment *sceneByFragmentConfig `yaml:"sceneByFragment"`
SceneByURL []*scrapeSceneByURLConfig `yaml:"sceneByURL"`
StashServer *stashServer `yaml:"stashServer"`
StashServer *stashServer `yaml:"stashServer"`
XPathScrapers xpathScrapers `yaml:"xPathScrapers"`
}
func loadScraperFromYAML(path string) (*scraperConfig, error) {