Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 12:54:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Unify scrape refactor (#1630)

Browse Source 
* Unify scraped types
* Make name fields optional
* Unify single scrape queries
* Change UI to use new interfaces
* Add multi scrape interfaces
* Use images instead of image
This commit is contained in:
WithoutPants
2021-09-07 11:54:22 +10:00
committed by GitHub
parent 04e146f290
commit 4625e1f955
40 changed files with 1035 additions and 781 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
pkg/scraper/json.go
View File

@@ -143,18 +143,9 @@ func (s *jsonScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedP
return nil, errors.New("scrapePerformerByFragment not supported for json scraper")
}
func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) {
storedScene, err := sceneFromUpdateFragment(scene, s.txnManager)
if err != nil {
return nil, err
}
if storedScene == nil {
return nil, errors.New("no scene found")
}
func (s *jsonScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) {
// construct the URL
queryURL := queryURLParametersFromScene(storedScene)
queryURL := queryURLParametersFromScene(scene)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
@@ -176,18 +167,13 @@ func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mod
return scraper.scrapeScene(q)
}
func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) {
storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager)
if err != nil {
return nil, err
}
if storedGallery == nil {
return nil, errors.New("no scene found")
}
func (s *jsonScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) {
return nil, errors.New("scrapeSceneByFragment not supported for json scraper")
}
func (s *jsonScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) {
// construct the URL
queryURL := queryURLParametersFromGallery(storedGallery)
queryURL := queryURLParametersFromGallery(gallery)
if s.scraper.QueryURLReplacements != nil {
queryURL.applyReplacements(s.scraper.QueryURLReplacements)
}
@@ -209,6 +195,10 @@ func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput)
return scraper.scrapeGallery(q)
}
func (s *jsonScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) {
return nil, errors.New("scrapeGalleryByFragment not supported for json scraper")
}
func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery {
return &jsonQuery{
doc: doc,