Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 21:04:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Use referer on xpath getImage, apply printHTML to subscraper also (#661)

Browse Source
This commit is contained in:
bnkai
2020-07-10 01:42:06 +03:00
committed by GitHub
parent e9141b5dfc
commit 56210cf456
2 changed files with 23 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/scraper/image.go
View File

@@ -64,7 +64,14 @@ func getImage(url string) (*string, error) {
} }
// assume is a URL for now // assume is a URL for now
// set the host of the URL as the referer
if req.URL.Scheme != "" {
req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host)
}
resp, err := client.Do(req) resp, err := client.Do(req)
if err != nil { if err != nil {
return nil, err return nil, err
} }

22
pkg/scraper/xpath.go
View File

@@ -25,6 +25,8 @@ import (
// configurable at some point. // configurable at some point.
const scrapeGetTimeout = time.Second * 30 const scrapeGetTimeout = time.Second * 30
var debugMode = false
type commonXPathConfig map[string]string type commonXPathConfig map[string]string
func (c commonXPathConfig) applyCommon(src string) string { func (c commonXPathConfig) applyCommon(src string) string {
@@ -259,7 +261,7 @@ func (c xpathScraperAttrConfig) applySubScraper(value string) string {
} }
logger.Debugf("Sub-scraping for: %s", value) logger.Debugf("Sub-scraping for: %s", value)
doc, err := loadURL(value, nil) doc, err := loadURL(value)
if err != nil { if err != nil {
logger.Warnf("Error getting URL '%s' for sub-scraper: %s", value, err.Error()) logger.Warnf("Error getting URL '%s' for sub-scraper: %s", value, err.Error())
@@ -588,7 +590,7 @@ func (r xPathResults) setKey(index int, key string, value string) xPathResults {
return r return r
} }
func loadURL(url string, c *scraperConfig) (*html.Node, error) { func loadURL(url string) (*html.Node, error) {
options := cookiejar.Options{ options := cookiejar.Options{
PublicSuffixList: publicsuffix.List, PublicSuffixList: publicsuffix.List,
} }
@@ -631,7 +633,7 @@ func loadURL(url string, c *scraperConfig) (*html.Node, error) {
ret, err := html.Parse(r) ret, err := html.Parse(r)
if err == nil && c != nil && c.DebugOptions != nil && c.DebugOptions.PrintHTML { if err == nil && debugMode {
var b bytes.Buffer var b bytes.Buffer
html.Render(&b, ret) html.Render(&b, ret)
logger.Infof("loadURL (%s) response: \n%s", url, b.String()) logger.Infof("loadURL (%s) response: \n%s", url, b.String())
@@ -647,7 +649,11 @@ func scrapePerformerURLXpath(c scraperTypeConfig, url string) (*models.ScrapedPe
return nil, errors.New("xpath scraper with name " + c.Scraper + " not found in config") return nil, errors.New("xpath scraper with name " + c.Scraper + " not found in config")
} }
doc, err := loadURL(url, c.scraperConfig) if c.scraperConfig != nil && c.scraperConfig.DebugOptions != nil && c.scraperConfig.DebugOptions.PrintHTML {
debugMode = true
}
doc, err := loadURL(url)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -663,7 +669,11 @@ func scrapeSceneURLXPath(c scraperTypeConfig, url string) (*models.ScrapedScene,
return nil, errors.New("xpath scraper with name " + c.Scraper + " not found in config") return nil, errors.New("xpath scraper with name " + c.Scraper + " not found in config")
} }
doc, err := loadURL(url, c.scraperConfig) if c.scraperConfig != nil && c.scraperConfig.DebugOptions != nil && c.scraperConfig.DebugOptions.PrintHTML {
debugMode = true
}
doc, err := loadURL(url)
if err != nil { if err != nil {
return nil, err return nil, err
@@ -687,7 +697,7 @@ func scrapePerformerNamesXPath(c scraperTypeConfig, name string) ([]*models.Scra
u := c.QueryURL u := c.QueryURL
u = strings.Replace(u, placeholder, escapedName, -1) u = strings.Replace(u, placeholder, escapedName, -1)
doc, err := loadURL(u, c.scraperConfig) doc, err := loadURL(u)
if err != nil { if err != nil {
return nil, err return nil, err