Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
bnkai
2020-08-04 03:42:40 +03:00
committed by GitHub
parent 1b4a9eed36
commit 4373f9bf01
284 changed files with 133250 additions and 54 deletions

View File

@@ -43,6 +43,9 @@ type config struct {
// Xpath scraping configurations
XPathScrapers mappedScrapers `yaml:"xPathScrapers"`
// Scraping driver options
DriverOptions *scraperDriverOptions `yaml:"driver"`
}
func (c config) validate() error {
@@ -135,6 +138,11 @@ type scraperDebugOptions struct {
PrintHTML bool `yaml:"printHTML"`
}
type scraperDriverOptions struct {
UseCDP bool `yaml:"useCDP"`
Sleep int `yaml:"sleep"`
}
func loadScraperFromYAML(id string, reader io.Reader) (*config, error) {
ret := &config{}