Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>
2025-12-17 12:24:38 +03:00 · 2020-08-04 03:42:40 +03:00
parent 1b4a9eed36
commit 4373f9bf01
284 changed files with 133250 additions and 54 deletions
--- a/vendor/github.com/chromedp/cdproto/fetch/fetch.go
+++ b/vendor/github.com/chromedp/cdproto/fetch/fetch.go
@@ -0,0 +1,356 @@
+// Package fetch provides the Chrome DevTools Protocol
+// commands, types, and events for the Fetch domain.
+//
+// A domain for letting clients substitute browser's network layer with
+// client code.
+//
+// Generated by the cdproto-gen command.
+package fetch
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"context"
+	"encoding/base64"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/io"
+	"github.com/chromedp/cdproto/network"
+)
+
+// DisableParams disables the fetch domain.
+type DisableParams struct{}
+
+// Disable disables the fetch domain.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-disable
+func Disable() *DisableParams {
+	return &DisableParams{}
+}
+
+// Do executes Fetch.disable against the provided context.
+func (p *DisableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandDisable, nil, nil)
+}
+
+// EnableParams enables issuing of requestPaused events. A request will be
+// paused until client calls one of failRequest, fulfillRequest or
+// continueRequest/continueWithAuth.
+type EnableParams struct {
+	Patterns           []*RequestPattern `json:"patterns,omitempty"`           // If specified, only requests matching any of these patterns will produce fetchRequested event and will be paused until clients response. If not set, all requests will be affected.
+	HandleAuthRequests bool              `json:"handleAuthRequests,omitempty"` // If true, authRequired events will be issued and requests will be paused expecting a call to continueWithAuth.
+}
+
+// Enable enables issuing of requestPaused events. A request will be paused
+// until client calls one of failRequest, fulfillRequest or
+// continueRequest/continueWithAuth.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-enable
+//
+// parameters:
+func Enable() *EnableParams {
+	return &EnableParams{}
+}
+
+// WithPatterns if specified, only requests matching any of these patterns
+// will produce fetchRequested event and will be paused until clients response.
+// If not set, all requests will be affected.
+func (p EnableParams) WithPatterns(patterns []*RequestPattern) *EnableParams {
+	p.Patterns = patterns
+	return &p
+}
+
+// WithHandleAuthRequests if true, authRequired events will be issued and
+// requests will be paused expecting a call to continueWithAuth.
+func (p EnableParams) WithHandleAuthRequests(handleAuthRequests bool) *EnableParams {
+	p.HandleAuthRequests = handleAuthRequests
+	return &p
+}
+
+// Do executes Fetch.enable against the provided context.
+func (p *EnableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandEnable, p, nil)
+}
+
+// FailRequestParams causes the request to fail with specified reason.
+type FailRequestParams struct {
+	RequestID   RequestID           `json:"requestId"`   // An id the client received in requestPaused event.
+	ErrorReason network.ErrorReason `json:"errorReason"` // Causes the request to fail with the given reason.
+}
+
+// FailRequest causes the request to fail with specified reason.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-failRequest
+//
+// parameters:
+//   requestID - An id the client received in requestPaused event.
+//   errorReason - Causes the request to fail with the given reason.
+func FailRequest(requestID RequestID, errorReason network.ErrorReason) *FailRequestParams {
+	return &FailRequestParams{
+		RequestID:   requestID,
+		ErrorReason: errorReason,
+	}
+}
+
+// Do executes Fetch.failRequest against the provided context.
+func (p *FailRequestParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandFailRequest, p, nil)
+}
+
+// FulfillRequestParams provides response to the request.
+type FulfillRequestParams struct {
+	RequestID             RequestID      `json:"requestId"`                       // An id the client received in requestPaused event.
+	ResponseCode          int64          `json:"responseCode"`                    // An HTTP response code.
+	ResponseHeaders       []*HeaderEntry `json:"responseHeaders,omitempty"`       // Response headers.
+	BinaryResponseHeaders string         `json:"binaryResponseHeaders,omitempty"` // Alternative way of specifying response headers as a \0-separated series of name: value pairs. Prefer the above method unless you need to represent some non-UTF8 values that can't be transmitted over the protocol as text.
+	Body                  string         `json:"body,omitempty"`                  // A response body.
+	ResponsePhrase        string         `json:"responsePhrase,omitempty"`        // A textual representation of responseCode. If absent, a standard phrase matching responseCode is used.
+}
+
+// FulfillRequest provides response to the request.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-fulfillRequest
+//
+// parameters:
+//   requestID - An id the client received in requestPaused event.
+//   responseCode - An HTTP response code.
+func FulfillRequest(requestID RequestID, responseCode int64) *FulfillRequestParams {
+	return &FulfillRequestParams{
+		RequestID:    requestID,
+		ResponseCode: responseCode,
+	}
+}
+
+// WithResponseHeaders response headers.
+func (p FulfillRequestParams) WithResponseHeaders(responseHeaders []*HeaderEntry) *FulfillRequestParams {
+	p.ResponseHeaders = responseHeaders
+	return &p
+}
+
+// WithBinaryResponseHeaders alternative way of specifying response headers
+// as a \0-separated series of name: value pairs. Prefer the above method unless
+// you need to represent some non-UTF8 values that can't be transmitted over the
+// protocol as text.
+func (p FulfillRequestParams) WithBinaryResponseHeaders(binaryResponseHeaders string) *FulfillRequestParams {
+	p.BinaryResponseHeaders = binaryResponseHeaders
+	return &p
+}
+
+// WithBody a response body.
+func (p FulfillRequestParams) WithBody(body string) *FulfillRequestParams {
+	p.Body = body
+	return &p
+}
+
+// WithResponsePhrase a textual representation of responseCode. If absent, a
+// standard phrase matching responseCode is used.
+func (p FulfillRequestParams) WithResponsePhrase(responsePhrase string) *FulfillRequestParams {
+	p.ResponsePhrase = responsePhrase
+	return &p
+}
+
+// Do executes Fetch.fulfillRequest against the provided context.
+func (p *FulfillRequestParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandFulfillRequest, p, nil)
+}
+
+// ContinueRequestParams continues the request, optionally modifying some of
+// its parameters.
+type ContinueRequestParams struct {
+	RequestID RequestID      `json:"requestId"`          // An id the client received in requestPaused event.
+	URL       string         `json:"url,omitempty"`      // If set, the request url will be modified in a way that's not observable by page.
+	Method    string         `json:"method,omitempty"`   // If set, the request method is overridden.
+	PostData  string         `json:"postData,omitempty"` // If set, overrides the post data in the request.
+	Headers   []*HeaderEntry `json:"headers,omitempty"`  // If set, overrides the request headers.
+}
+
+// ContinueRequest continues the request, optionally modifying some of its
+// parameters.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-continueRequest
+//
+// parameters:
+//   requestID - An id the client received in requestPaused event.
+func ContinueRequest(requestID RequestID) *ContinueRequestParams {
+	return &ContinueRequestParams{
+		RequestID: requestID,
+	}
+}
+
+// WithURL if set, the request url will be modified in a way that's not
+// observable by page.
+func (p ContinueRequestParams) WithURL(url string) *ContinueRequestParams {
+	p.URL = url
+	return &p
+}
+
+// WithMethod if set, the request method is overridden.
+func (p ContinueRequestParams) WithMethod(method string) *ContinueRequestParams {
+	p.Method = method
+	return &p
+}
+
+// WithPostData if set, overrides the post data in the request.
+func (p ContinueRequestParams) WithPostData(postData string) *ContinueRequestParams {
+	p.PostData = postData
+	return &p
+}
+
+// WithHeaders if set, overrides the request headers.
+func (p ContinueRequestParams) WithHeaders(headers []*HeaderEntry) *ContinueRequestParams {
+	p.Headers = headers
+	return &p
+}
+
+// Do executes Fetch.continueRequest against the provided context.
+func (p *ContinueRequestParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandContinueRequest, p, nil)
+}
+
+// ContinueWithAuthParams continues a request supplying authChallengeResponse
+// following authRequired event.
+type ContinueWithAuthParams struct {
+	RequestID             RequestID              `json:"requestId"`             // An id the client received in authRequired event.
+	AuthChallengeResponse *AuthChallengeResponse `json:"authChallengeResponse"` // Response to  with an authChallenge.
+}
+
+// ContinueWithAuth continues a request supplying authChallengeResponse
+// following authRequired event.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-continueWithAuth
+//
+// parameters:
+//   requestID - An id the client received in authRequired event.
+//   authChallengeResponse - Response to  with an authChallenge.
+func ContinueWithAuth(requestID RequestID, authChallengeResponse *AuthChallengeResponse) *ContinueWithAuthParams {
+	return &ContinueWithAuthParams{
+		RequestID:             requestID,
+		AuthChallengeResponse: authChallengeResponse,
+	}
+}
+
+// Do executes Fetch.continueWithAuth against the provided context.
+func (p *ContinueWithAuthParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandContinueWithAuth, p, nil)
+}
+
+// GetResponseBodyParams causes the body of the response to be received from
+// the server and returned as a single string. May only be issued for a request
+// that is paused in the Response stage and is mutually exclusive with
+// takeResponseBodyForInterceptionAsStream. Calling other methods that affect
+// the request or disabling fetch domain before body is received results in an
+// undefined behavior.
+type GetResponseBodyParams struct {
+	RequestID RequestID `json:"requestId"` // Identifier for the intercepted request to get body for.
+}
+
+// GetResponseBody causes the body of the response to be received from the
+// server and returned as a single string. May only be issued for a request that
+// is paused in the Response stage and is mutually exclusive with
+// takeResponseBodyForInterceptionAsStream. Calling other methods that affect
+// the request or disabling fetch domain before body is received results in an
+// undefined behavior.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-getResponseBody
+//
+// parameters:
+//   requestID - Identifier for the intercepted request to get body for.
+func GetResponseBody(requestID RequestID) *GetResponseBodyParams {
+	return &GetResponseBodyParams{
+		RequestID: requestID,
+	}
+}
+
+// GetResponseBodyReturns return values.
+type GetResponseBodyReturns struct {
+	Body          string `json:"body,omitempty"`          // Response body.
+	Base64encoded bool   `json:"base64Encoded,omitempty"` // True, if content was sent as base64.
+}
+
+// Do executes Fetch.getResponseBody against the provided context.
+//
+// returns:
+//   body - Response body.
+func (p *GetResponseBodyParams) Do(ctx context.Context) (body []byte, err error) {
+	// execute
+	var res GetResponseBodyReturns
+	err = cdp.Execute(ctx, CommandGetResponseBody, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	// decode
+	var dec []byte
+	if res.Base64encoded {
+		dec, err = base64.StdEncoding.DecodeString(res.Body)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		dec = []byte(res.Body)
+	}
+	return dec, nil
+}
+
+// TakeResponseBodyAsStreamParams returns a handle to the stream representing
+// the response body. The request must be paused in the HeadersReceived stage.
+// Note that after this command the request can't be continued as is -- client
+// either needs to cancel it or to provide the response body. The stream only
+// supports sequential read, IO.read will fail if the position is specified.
+// This method is mutually exclusive with getResponseBody. Calling other methods
+// that affect the request or disabling fetch domain before body is received
+// results in an undefined behavior.
+type TakeResponseBodyAsStreamParams struct {
+	RequestID RequestID `json:"requestId"`
+}
+
+// TakeResponseBodyAsStream returns a handle to the stream representing the
+// response body. The request must be paused in the HeadersReceived stage. Note
+// that after this command the request can't be continued as is -- client either
+// needs to cancel it or to provide the response body. The stream only supports
+// sequential read, IO.read will fail if the position is specified. This method
+// is mutually exclusive with getResponseBody. Calling other methods that affect
+// the request or disabling fetch domain before body is received results in an
+// undefined behavior.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Fetch#method-takeResponseBodyAsStream
+//
+// parameters:
+//   requestID
+func TakeResponseBodyAsStream(requestID RequestID) *TakeResponseBodyAsStreamParams {
+	return &TakeResponseBodyAsStreamParams{
+		RequestID: requestID,
+	}
+}
+
+// TakeResponseBodyAsStreamReturns return values.
+type TakeResponseBodyAsStreamReturns struct {
+	Stream io.StreamHandle `json:"stream,omitempty"`
+}
+
+// Do executes Fetch.takeResponseBodyAsStream against the provided context.
+//
+// returns:
+//   stream
+func (p *TakeResponseBodyAsStreamParams) Do(ctx context.Context) (stream io.StreamHandle, err error) {
+	// execute
+	var res TakeResponseBodyAsStreamReturns
+	err = cdp.Execute(ctx, CommandTakeResponseBodyAsStream, p, &res)
+	if err != nil {
+		return "", err
+	}
+
+	return res.Stream, nil
+}
+
+// Command names.
+const (
+	CommandDisable                  = "Fetch.disable"
+	CommandEnable                   = "Fetch.enable"
+	CommandFailRequest              = "Fetch.failRequest"
+	CommandFulfillRequest           = "Fetch.fulfillRequest"
+	CommandContinueRequest          = "Fetch.continueRequest"
+	CommandContinueWithAuth         = "Fetch.continueWithAuth"
+	CommandGetResponseBody          = "Fetch.getResponseBody"
+	CommandTakeResponseBodyAsStream = "Fetch.takeResponseBodyAsStream"
+)