Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-17 12:24:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Feature: Support inputURL and inputHostname in scrapers (#6250)

Browse Source
This commit is contained in:
Gykes
2025-11-09 20:00:47 -08:00
committed by GitHub
parent f434c1f529
commit 678b3de7c8
4 changed files with 102 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pkg/scraper/json.go
View File

@@ -80,7 +80,7 @@ func (s *jsonScraper) scrapeByURL(ctx context.Context, url string, ty ScrapeCont
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, u)
// if these just return the return values from scraper.scrape* functions then
// it ends up returning ScrapedContent(nil) rather than nil
switch ty {
@@ -140,7 +140,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty ScrapeCo
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, url)
q.setType(SearchQuery)
var content []ScrapedContent
@@ -192,7 +192,7 @@ func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scen
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, url)
return scraper.scrapeScene(ctx, q)
}
@@ -227,7 +227,7 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, url)
return scraper.scrapeScene(ctx, q)
}
@@ -251,7 +251,7 @@ func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Imag
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, url)
return scraper.scrapeImage(ctx, q)
}
@@ -275,14 +275,15 @@ func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *model
return nil, err
}
q := s.getJsonQuery(doc)
q := s.getJsonQuery(doc, url)
return scraper.scrapeGallery(ctx, q)
}
func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery {
func (s *jsonScraper) getJsonQuery(doc string, url string) *jsonQuery {
return &jsonQuery{
doc: doc,
scraper: s,
url: url,
}
}
@@ -290,6 +291,7 @@ type jsonQuery struct {
doc string
scraper *jsonScraper
queryType QueryType
url string
}
func (q *jsonQuery) getType() QueryType {
@@ -300,6 +302,10 @@ func (q *jsonQuery) setType(t QueryType) {
q.queryType = t
}
func (q *jsonQuery) getURL() string {
return q.url
}
func (q *jsonQuery) runQuery(selector string) ([]string, error) {
value := gjson.Get(q.doc, selector)
@@ -331,5 +337,5 @@ func (q *jsonQuery) subScrape(ctx context.Context, value string) mappedQuery {
return nil
}
return q.scraper.getJsonQuery(doc)
return q.scraper.getJsonQuery(doc, value)
}