Skip cleaning for search by name scrape queries (#2059)

* Skip pp for search by name queries
* upgrade htmlquery
This commit is contained in:
bnkai
2021-12-16 02:18:39 +02:00
committed by GitHub
parent 439c338049
commit 66dd239732
34 changed files with 10925 additions and 10665 deletions

View File

@@ -100,6 +100,7 @@ func (s *xpathScraper) scrapeByName(ctx context.Context, name string, ty models.
}
q := s.getXPathQuery(doc)
q.setType(SearchQuery)
var content []models.ScrapedContent
switch ty {
@@ -238,8 +239,17 @@ func (s *xpathScraper) getXPathQuery(doc *html.Node) *xpathQuery {
}
type xpathQuery struct {
doc *html.Node
scraper *xpathScraper
doc *html.Node
scraper *xpathScraper
queryType QueryType
}
func (q *xpathQuery) getType() QueryType {
return q.queryType
}
func (q *xpathQuery) setType(t QueryType) {
q.queryType = t
}
func (q *xpathQuery) runQuery(selector string) ([]string, error) {