Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
This commit is contained in:
bnkai
2020-08-04 03:42:40 +03:00
committed by GitHub
parent 1b4a9eed36
commit 4373f9bf01
284 changed files with 133250 additions and 54 deletions

390
vendor/github.com/chromedp/cdproto/io/easyjson.go generated vendored Normal file
View File

@@ -0,0 +1,390 @@
// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT.
package io
import (
json "encoding/json"
runtime "github.com/chromedp/cdproto/runtime"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// suppress unused package warning
var (
_ *json.RawMessage
_ *jlexer.Lexer
_ *jwriter.Writer
_ easyjson.Marshaler
)
func easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo(in *jlexer.Lexer, out *ResolveBlobReturns) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "uuid":
out.UUID = string(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo(out *jwriter.Writer, in ResolveBlobReturns) {
out.RawByte('{')
first := true
_ = first
if in.UUID != "" {
const prefix string = ",\"uuid\":"
first = false
out.RawString(prefix[1:])
out.String(string(in.UUID))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v ResolveBlobReturns) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v ResolveBlobReturns) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *ResolveBlobReturns) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *ResolveBlobReturns) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo(l, v)
}
func easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo1(in *jlexer.Lexer, out *ResolveBlobParams) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "objectId":
out.ObjectID = runtime.RemoteObjectID(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo1(out *jwriter.Writer, in ResolveBlobParams) {
out.RawByte('{')
first := true
_ = first
{
const prefix string = ",\"objectId\":"
out.RawString(prefix[1:])
out.String(string(in.ObjectID))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v ResolveBlobParams) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo1(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v ResolveBlobParams) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo1(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *ResolveBlobParams) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo1(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *ResolveBlobParams) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo1(l, v)
}
func easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo2(in *jlexer.Lexer, out *ReadReturns) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "base64Encoded":
out.Base64encoded = bool(in.Bool())
case "data":
out.Data = string(in.String())
case "eof":
out.EOF = bool(in.Bool())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo2(out *jwriter.Writer, in ReadReturns) {
out.RawByte('{')
first := true
_ = first
if in.Base64encoded {
const prefix string = ",\"base64Encoded\":"
first = false
out.RawString(prefix[1:])
out.Bool(bool(in.Base64encoded))
}
if in.Data != "" {
const prefix string = ",\"data\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Data))
}
if in.EOF {
const prefix string = ",\"eof\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.Bool(bool(in.EOF))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v ReadReturns) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo2(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v ReadReturns) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo2(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *ReadReturns) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo2(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *ReadReturns) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo2(l, v)
}
func easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo3(in *jlexer.Lexer, out *ReadParams) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "handle":
out.Handle = StreamHandle(in.String())
case "offset":
out.Offset = int64(in.Int64())
case "size":
out.Size = int64(in.Int64())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo3(out *jwriter.Writer, in ReadParams) {
out.RawByte('{')
first := true
_ = first
{
const prefix string = ",\"handle\":"
out.RawString(prefix[1:])
out.String(string(in.Handle))
}
if in.Offset != 0 {
const prefix string = ",\"offset\":"
out.RawString(prefix)
out.Int64(int64(in.Offset))
}
if in.Size != 0 {
const prefix string = ",\"size\":"
out.RawString(prefix)
out.Int64(int64(in.Size))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v ReadParams) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo3(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v ReadParams) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo3(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *ReadParams) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo3(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *ReadParams) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo3(l, v)
}
func easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo4(in *jlexer.Lexer, out *CloseParams) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "handle":
out.Handle = StreamHandle(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo4(out *jwriter.Writer, in CloseParams) {
out.RawByte('{')
first := true
_ = first
{
const prefix string = ",\"handle\":"
out.RawString(prefix[1:])
out.String(string(in.Handle))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v CloseParams) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo4(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v CloseParams) MarshalEasyJSON(w *jwriter.Writer) {
easyjsonC5a4559bEncodeGithubComChromedpCdprotoIo4(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *CloseParams) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo4(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *CloseParams) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjsonC5a4559bDecodeGithubComChromedpCdprotoIo4(l, v)
}

140
vendor/github.com/chromedp/cdproto/io/io.go generated vendored Normal file
View File

@@ -0,0 +1,140 @@
// Package io provides the Chrome DevTools Protocol
// commands, types, and events for the IO domain.
//
// Input/Output operations for streams produced by DevTools.
//
// Generated by the cdproto-gen command.
package io
// Code generated by cdproto-gen. DO NOT EDIT.
import (
"context"
"github.com/chromedp/cdproto/cdp"
"github.com/chromedp/cdproto/runtime"
)
// CloseParams close the stream, discard any temporary backing storage.
type CloseParams struct {
Handle StreamHandle `json:"handle"` // Handle of the stream to close.
}
// Close close the stream, discard any temporary backing storage.
//
// See: https://chromedevtools.github.io/devtools-protocol/tot/IO#method-close
//
// parameters:
// handle - Handle of the stream to close.
func Close(handle StreamHandle) *CloseParams {
return &CloseParams{
Handle: handle,
}
}
// Do executes IO.close against the provided context.
func (p *CloseParams) Do(ctx context.Context) (err error) {
return cdp.Execute(ctx, CommandClose, p, nil)
}
// ReadParams read a chunk of the stream.
type ReadParams struct {
Handle StreamHandle `json:"handle"` // Handle of the stream to read.
Offset int64 `json:"offset,omitempty"` // Seek to the specified offset before reading (if not specificed, proceed with offset following the last read). Some types of streams may only support sequential reads.
Size int64 `json:"size,omitempty"` // Maximum number of bytes to read (left upon the agent discretion if not specified).
}
// Read read a chunk of the stream.
//
// See: https://chromedevtools.github.io/devtools-protocol/tot/IO#method-read
//
// parameters:
// handle - Handle of the stream to read.
func Read(handle StreamHandle) *ReadParams {
return &ReadParams{
Handle: handle,
}
}
// WithOffset seek to the specified offset before reading (if not specificed,
// proceed with offset following the last read). Some types of streams may only
// support sequential reads.
func (p ReadParams) WithOffset(offset int64) *ReadParams {
p.Offset = offset
return &p
}
// WithSize maximum number of bytes to read (left upon the agent discretion
// if not specified).
func (p ReadParams) WithSize(size int64) *ReadParams {
p.Size = size
return &p
}
// ReadReturns return values.
type ReadReturns struct {
Base64encoded bool `json:"base64Encoded,omitempty"` // Set if the data is base64-encoded
Data string `json:"data,omitempty"` // Data that were read.
EOF bool `json:"eof,omitempty"` // Set if the end-of-file condition occurred while reading.
}
// Do executes IO.read against the provided context.
//
// returns:
// data - Data that were read.
// eof - Set if the end-of-file condition occurred while reading.
func (p *ReadParams) Do(ctx context.Context) (data string, eof bool, err error) {
// execute
var res ReadReturns
err = cdp.Execute(ctx, CommandRead, p, &res)
if err != nil {
return "", false, err
}
return res.Data, res.EOF, nil
}
// ResolveBlobParams return UUID of Blob object specified by a remote object
// id.
type ResolveBlobParams struct {
ObjectID runtime.RemoteObjectID `json:"objectId"` // Object id of a Blob object wrapper.
}
// ResolveBlob return UUID of Blob object specified by a remote object id.
//
// See: https://chromedevtools.github.io/devtools-protocol/tot/IO#method-resolveBlob
//
// parameters:
// objectID - Object id of a Blob object wrapper.
func ResolveBlob(objectID runtime.RemoteObjectID) *ResolveBlobParams {
return &ResolveBlobParams{
ObjectID: objectID,
}
}
// ResolveBlobReturns return values.
type ResolveBlobReturns struct {
UUID string `json:"uuid,omitempty"` // UUID of the specified Blob.
}
// Do executes IO.resolveBlob against the provided context.
//
// returns:
// uuid - UUID of the specified Blob.
func (p *ResolveBlobParams) Do(ctx context.Context) (uuid string, err error) {
// execute
var res ResolveBlobReturns
err = cdp.Execute(ctx, CommandResolveBlob, p, &res)
if err != nil {
return "", err
}
return res.UUID, nil
}
// Command names.
const (
CommandClose = "IO.close"
CommandRead = "IO.read"
CommandResolveBlob = "IO.resolveBlob"
)

14
vendor/github.com/chromedp/cdproto/io/types.go generated vendored Normal file
View File

@@ -0,0 +1,14 @@
package io
// Code generated by cdproto-gen. DO NOT EDIT.
// StreamHandle this is either obtained from another method or specified as
// blob:<uuid> where <uuid> is an UUID of a Blob.
//
// See: https://chromedevtools.github.io/devtools-protocol/tot/IO#type-StreamHandle
type StreamHandle string
// String returns the StreamHandle as string value.
func (t StreamHandle) String() string {
return string(t)
}