Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 04:14:39 +03:00
Code Issues Packages Projects Releases Wiki Activity

Unify scrape refactor (#1630)

Browse Source 
* Unify scraped types
* Make name fields optional
* Unify single scrape queries
* Change UI to use new interfaces
* Add multi scrape interfaces
* Use images instead of image
This commit is contained in:
WithoutPants
2021-09-07 11:54:22 +10:00
committed by GitHub
parent 04e146f290
commit 4625e1f955
40 changed files with 1035 additions and 781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
pkg/scraper/xpath.go
View File

@@ -124,18 +124,9 @@ func (s *xpathScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped
return nil, errors.New("scrapePerformerByFragment not supported for xpath scraper")
}
func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) {
storedScene, err := sceneFromUpdateFragment(scene, s.txnManager)
if err != nil {
return nil, err
}
if storedScene == nil {
return nil, errors.New("no scene found")
}
func (s *xpathScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) {
// construct the URL
queryURL := queryURLParametersFromScene(storedScene)
queryURL := queryURLParametersFromScene(scene)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
@@ -157,18 +148,13 @@ func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo
return scraper.scrapeScene(q)
}
func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) {
storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager)
if err != nil {
return nil, err
}
if storedGallery == nil {
return nil, errors.New("no scene found")
}
func (s *xpathScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) {
return nil, errors.New("scrapeSceneByFragment not supported for xpath scraper")
}
func (s *xpathScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) {
// construct the URL
queryURL := queryURLParametersFromGallery(storedGallery)
queryURL := queryURLParametersFromGallery(gallery)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
@@ -190,6 +176,10 @@ func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput
return scraper.scrapeGallery(q)
}
func (s *xpathScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) {
return nil, errors.New("scrapeGalleryByFragment not supported for xpath scraper")
}
func (s *xpathScraper) loadURL(url string) (*html.Node, error) {
r, err := loadURL(url, s.config, s.globalConfig)
if err != nil {