Add gallery scraping (#862)

This commit is contained in:
SpedNSFW
2020-10-21 09:24:32 +11:00
committed by GitHub
parent 872bb70f6e
commit 147d0067f5
19 changed files with 1296 additions and 1 deletions

View File

@@ -88,6 +88,16 @@ func (s *jsonScraper) scrapeSceneByURL(url string) (*models.ScrapedScene, error)
return scraper.scrapeScene(q)
}
func (s *jsonScraper) scrapeGalleryByURL(url string) (*models.ScrapedGallery, error) {
doc, scraper, err := s.scrapeURL(url)
if err != nil {
return nil, err
}
q := s.getJsonQuery(doc)
return scraper.scrapeGallery(q)
}
func (s *jsonScraper) scrapeMovieByURL(url string) (*models.ScrapedMovie, error) {
doc, scraper, err := s.scrapeURL(url)
if err != nil {
@@ -156,6 +166,34 @@ func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mod
return scraper.scrapeScene(q)
}
func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) {
storedGallery, err := galleryFromUpdateFragment(gallery)
if err != nil {
return nil, err
}
if storedGallery == nil {
return nil, errors.New("no scene found")
}
url := constructGalleryURL(s.scraper.QueryURL, storedGallery)
scraper := s.getJsonScraper()
if scraper == nil {
return nil, errors.New("json scraper with name " + s.scraper.Scraper + " not found in config")
}
doc, err := s.loadURL(url)
if err != nil {
return nil, err
}
q := s.getJsonQuery(doc)
return scraper.scrapeGallery(q)
}
func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery {
return &jsonQuery{
doc: doc,