Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

vendor/github.com/chromedp/cdproto/tracing/events.go

@@ -0,0 +1,37 @@
+package tracing
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"github.com/chromedp/cdproto/io"
+	"github.com/mailru/easyjson"
+)
+
+// EventBufferUsage [no description].
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#event-bufferUsage
+type EventBufferUsage struct {
+	PercentFull float64 `json:"percentFull,omitempty"` // A number in range [0..1] that indicates the used size of event buffer as a fraction of its total size.
+	EventCount  float64 `json:"eventCount,omitempty"`  // An approximate number of events in the trace log.
+	Value       float64 `json:"value,omitempty"`       // A number in range [0..1] that indicates the used size of event buffer as a fraction of its total size.
+}
+
+// EventDataCollected contains an bucket of collected trace events. When
+// tracing is stopped collected events will be send as a sequence of
+// dataCollected events followed by tracingComplete event.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#event-dataCollected
+type EventDataCollected struct {
+	Value []easyjson.RawMessage `json:"value"`
+}
+
+// EventTracingComplete signals that tracing is stopped and there is no trace
+// buffers pending flush, all data were delivered via dataCollected events.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#event-tracingComplete
+type EventTracingComplete struct {
+	DataLossOccurred  bool              `json:"dataLossOccurred"`            // Indicates whether some trace data is known to have been lost, e.g. because the trace ring buffer wrapped around.
+	Stream            io.StreamHandle   `json:"stream,omitempty"`            // A handle of the stream that holds resulting trace data.
+	TraceFormat       StreamFormat      `json:"traceFormat,omitempty"`       // Trace data format of returned stream.
+	StreamCompression StreamCompression `json:"streamCompression,omitempty"` // Compression format of returned stream.
+}

vendor/github.com/chromedp/cdproto/tracing/tracing.go

@@ -0,0 +1,189 @@
+// Package tracing provides the Chrome DevTools Protocol
+// commands, types, and events for the Tracing domain.
+//
+// Generated by the cdproto-gen command.
+package tracing
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"context"
+
+	"github.com/chromedp/cdproto/cdp"
+)
+
+// EndParams stop trace events collection.
+type EndParams struct{}
+
+// End stop trace events collection.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-end
+func End() *EndParams {
+	return &EndParams{}
+}
+
+// Do executes Tracing.end against the provided context.
+func (p *EndParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandEnd, nil, nil)
+}
+
+// GetCategoriesParams gets supported tracing categories.
+type GetCategoriesParams struct{}
+
+// GetCategories gets supported tracing categories.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-getCategories
+func GetCategories() *GetCategoriesParams {
+	return &GetCategoriesParams{}
+}
+
+// GetCategoriesReturns return values.
+type GetCategoriesReturns struct {
+	Categories []string `json:"categories,omitempty"` // A list of supported tracing categories.
+}
+
+// Do executes Tracing.getCategories against the provided context.
+//
+// returns:
+//   categories - A list of supported tracing categories.
+func (p *GetCategoriesParams) Do(ctx context.Context) (categories []string, err error) {
+	// execute
+	var res GetCategoriesReturns
+	err = cdp.Execute(ctx, CommandGetCategories, nil, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Categories, nil
+}
+
+// RecordClockSyncMarkerParams record a clock sync marker in the trace.
+type RecordClockSyncMarkerParams struct {
+	SyncID string `json:"syncId"` // The ID of this clock sync marker
+}
+
+// RecordClockSyncMarker record a clock sync marker in the trace.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-recordClockSyncMarker
+//
+// parameters:
+//   syncID - The ID of this clock sync marker
+func RecordClockSyncMarker(syncID string) *RecordClockSyncMarkerParams {
+	return &RecordClockSyncMarkerParams{
+		SyncID: syncID,
+	}
+}
+
+// Do executes Tracing.recordClockSyncMarker against the provided context.
+func (p *RecordClockSyncMarkerParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandRecordClockSyncMarker, p, nil)
+}
+
+// RequestMemoryDumpParams request a global memory dump.
+type RequestMemoryDumpParams struct {
+	Deterministic bool `json:"deterministic,omitempty"` // Enables more deterministic results by forcing garbage collection
+}
+
+// RequestMemoryDump request a global memory dump.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-requestMemoryDump
+//
+// parameters:
+func RequestMemoryDump() *RequestMemoryDumpParams {
+	return &RequestMemoryDumpParams{}
+}
+
+// WithDeterministic enables more deterministic results by forcing garbage
+// collection.
+func (p RequestMemoryDumpParams) WithDeterministic(deterministic bool) *RequestMemoryDumpParams {
+	p.Deterministic = deterministic
+	return &p
+}
+
+// RequestMemoryDumpReturns return values.
+type RequestMemoryDumpReturns struct {
+	DumpGUID string `json:"dumpGuid,omitempty"` // GUID of the resulting global memory dump.
+	Success  bool   `json:"success,omitempty"`  // True iff the global memory dump succeeded.
+}
+
+// Do executes Tracing.requestMemoryDump against the provided context.
+//
+// returns:
+//   dumpGUID - GUID of the resulting global memory dump.
+//   success - True iff the global memory dump succeeded.
+func (p *RequestMemoryDumpParams) Do(ctx context.Context) (dumpGUID string, success bool, err error) {
+	// execute
+	var res RequestMemoryDumpReturns
+	err = cdp.Execute(ctx, CommandRequestMemoryDump, p, &res)
+	if err != nil {
+		return "", false, err
+	}
+
+	return res.DumpGUID, res.Success, nil
+}
+
+// StartParams start trace events collection.
+type StartParams struct {
+	BufferUsageReportingInterval float64           `json:"bufferUsageReportingInterval,omitempty"` // If set, the agent will issue bufferUsage events at this interval, specified in milliseconds
+	TransferMode                 TransferMode      `json:"transferMode,omitempty"`                 // Whether to report trace events as series of dataCollected events or to save trace to a stream (defaults to ReportEvents).
+	StreamFormat                 StreamFormat      `json:"streamFormat,omitempty"`                 // Trace data format to use. This only applies when using ReturnAsStream transfer mode (defaults to json).
+	StreamCompression            StreamCompression `json:"streamCompression,omitempty"`            // Compression format to use. This only applies when using ReturnAsStream transfer mode (defaults to none)
+	TraceConfig                  *TraceConfig      `json:"traceConfig,omitempty"`
+}
+
+// Start start trace events collection.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-start
+//
+// parameters:
+func Start() *StartParams {
+	return &StartParams{}
+}
+
+// WithBufferUsageReportingInterval if set, the agent will issue bufferUsage
+// events at this interval, specified in milliseconds.
+func (p StartParams) WithBufferUsageReportingInterval(bufferUsageReportingInterval float64) *StartParams {
+	p.BufferUsageReportingInterval = bufferUsageReportingInterval
+	return &p
+}
+
+// WithTransferMode whether to report trace events as series of dataCollected
+// events or to save trace to a stream (defaults to ReportEvents).
+func (p StartParams) WithTransferMode(transferMode TransferMode) *StartParams {
+	p.TransferMode = transferMode
+	return &p
+}
+
+// WithStreamFormat trace data format to use. This only applies when using
+// ReturnAsStream transfer mode (defaults to json).
+func (p StartParams) WithStreamFormat(streamFormat StreamFormat) *StartParams {
+	p.StreamFormat = streamFormat
+	return &p
+}
+
+// WithStreamCompression compression format to use. This only applies when
+// using ReturnAsStream transfer mode (defaults to none).
+func (p StartParams) WithStreamCompression(streamCompression StreamCompression) *StartParams {
+	p.StreamCompression = streamCompression
+	return &p
+}
+
+// WithTraceConfig [no description].
+func (p StartParams) WithTraceConfig(traceConfig *TraceConfig) *StartParams {
+	p.TraceConfig = traceConfig
+	return &p
+}
+
+// Do executes Tracing.start against the provided context.
+func (p *StartParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandStart, p, nil)
+}
+
+// Command names.
+const (
+	CommandEnd                   = "Tracing.end"
+	CommandGetCategories         = "Tracing.getCategories"
+	CommandRecordClockSyncMarker = "Tracing.recordClockSyncMarker"
+	CommandRequestMemoryDump     = "Tracing.requestMemoryDump"
+	CommandStart                 = "Tracing.start"
+)

vendor/github.com/chromedp/cdproto/tracing/types.go

@@ -0,0 +1,216 @@
+package tracing
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"errors"
+
+	"github.com/mailru/easyjson"
+	"github.com/mailru/easyjson/jlexer"
+	"github.com/mailru/easyjson/jwriter"
+)
+
+// MemoryDumpConfig configuration for memory dump. Used only when
+// "memory-infra" category is enabled.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#type-MemoryDumpConfig
+type MemoryDumpConfig struct{}
+
+// TraceConfig [no description].
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#type-TraceConfig
+type TraceConfig struct {
+	RecordMode           RecordMode        `json:"recordMode,omitempty"`           // Controls how the trace buffer stores data.
+	EnableSampling       bool              `json:"enableSampling,omitempty"`       // Turns on JavaScript stack sampling.
+	EnableSystrace       bool              `json:"enableSystrace,omitempty"`       // Turns on system tracing.
+	EnableArgumentFilter bool              `json:"enableArgumentFilter,omitempty"` // Turns on argument filter.
+	IncludedCategories   []string          `json:"includedCategories,omitempty"`   // Included category filters.
+	ExcludedCategories   []string          `json:"excludedCategories,omitempty"`   // Excluded category filters.
+	SyntheticDelays      []string          `json:"syntheticDelays,omitempty"`      // Configuration to synthesize the delays in tracing.
+	MemoryDumpConfig     *MemoryDumpConfig `json:"memoryDumpConfig,omitempty"`     // Configuration for memory dump triggers. Used only when "memory-infra" category is enabled.
+}
+
+// StreamFormat data format of a trace. Can be either the legacy JSON format
+// or the protocol buffer format. Note that the JSON format will be deprecated
+// soon.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#type-StreamFormat
+type StreamFormat string
+
+// String returns the StreamFormat as string value.
+func (t StreamFormat) String() string {
+	return string(t)
+}
+
+// StreamFormat values.
+const (
+	StreamFormatJSON  StreamFormat = "json"
+	StreamFormatProto StreamFormat = "proto"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t StreamFormat) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t StreamFormat) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *StreamFormat) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch StreamFormat(in.String()) {
+	case StreamFormatJSON:
+		*t = StreamFormatJSON
+	case StreamFormatProto:
+		*t = StreamFormatProto
+
+	default:
+		in.AddError(errors.New("unknown StreamFormat value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *StreamFormat) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}
+
+// StreamCompression compression type to use for traces returned via streams.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#type-StreamCompression
+type StreamCompression string
+
+// String returns the StreamCompression as string value.
+func (t StreamCompression) String() string {
+	return string(t)
+}
+
+// StreamCompression values.
+const (
+	StreamCompressionNone StreamCompression = "none"
+	StreamCompressionGzip StreamCompression = "gzip"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t StreamCompression) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t StreamCompression) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *StreamCompression) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch StreamCompression(in.String()) {
+	case StreamCompressionNone:
+		*t = StreamCompressionNone
+	case StreamCompressionGzip:
+		*t = StreamCompressionGzip
+
+	default:
+		in.AddError(errors.New("unknown StreamCompression value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *StreamCompression) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}
+
+// RecordMode controls how the trace buffer stores data.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#type-TraceConfig
+type RecordMode string
+
+// String returns the RecordMode as string value.
+func (t RecordMode) String() string {
+	return string(t)
+}
+
+// RecordMode values.
+const (
+	RecordModeRecordUntilFull        RecordMode = "recordUntilFull"
+	RecordModeRecordContinuously     RecordMode = "recordContinuously"
+	RecordModeRecordAsMuchAsPossible RecordMode = "recordAsMuchAsPossible"
+	RecordModeEchoToConsole          RecordMode = "echoToConsole"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t RecordMode) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t RecordMode) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *RecordMode) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch RecordMode(in.String()) {
+	case RecordModeRecordUntilFull:
+		*t = RecordModeRecordUntilFull
+	case RecordModeRecordContinuously:
+		*t = RecordModeRecordContinuously
+	case RecordModeRecordAsMuchAsPossible:
+		*t = RecordModeRecordAsMuchAsPossible
+	case RecordModeEchoToConsole:
+		*t = RecordModeEchoToConsole
+
+	default:
+		in.AddError(errors.New("unknown RecordMode value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *RecordMode) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}
+
+// TransferMode whether to report trace events as series of dataCollected
+// events or to save trace to a stream (defaults to ReportEvents).
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Tracing#method-start
+type TransferMode string
+
+// String returns the TransferMode as string value.
+func (t TransferMode) String() string {
+	return string(t)
+}
+
+// TransferMode values.
+const (
+	TransferModeReportEvents   TransferMode = "ReportEvents"
+	TransferModeReturnAsStream TransferMode = "ReturnAsStream"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t TransferMode) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t TransferMode) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *TransferMode) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch TransferMode(in.String()) {
+	case TransferModeReportEvents:
+		*t = TransferModeReportEvents
+	case TransferModeReturnAsStream:
+		*t = TransferModeReturnAsStream
+
+	default:
+		in.AddError(errors.New("unknown TransferMode value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *TransferMode) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}