Skip cleaning for search by name scrape queries (#2059)

* Skip pp for search by name queries
* upgrade htmlquery
This commit is contained in:
bnkai
2021-12-16 02:18:39 +02:00
committed by GitHub
parent 439c338049
commit 66dd239732
34 changed files with 10925 additions and 10665 deletions

View File

@@ -119,6 +119,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty models.S
}
q := s.getJsonQuery(doc)
q.setType(SearchQuery)
var content []models.ScrapedContent
switch ty {
@@ -240,8 +241,17 @@ func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery {
}
type jsonQuery struct {
doc string
scraper *jsonScraper
doc string
scraper *jsonScraper
queryType QueryType
}
func (q *jsonQuery) getType() QueryType {
return q.queryType
}
func (q *jsonQuery) setType(t QueryType) {
q.queryType = t
}
func (q *jsonQuery) runQuery(selector string) ([]string, error) {