Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

vendor/github.com/chromedp/cdproto/layertree/events.go

@@ -0,0 +1,22 @@
+package layertree
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"github.com/chromedp/cdproto/dom"
+)
+
+// EventLayerPainted [no description].
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#event-layerPainted
+type EventLayerPainted struct {
+	LayerID LayerID   `json:"layerId"` // The id of the painted layer.
+	Clip    *dom.Rect `json:"clip"`    // Clip rectangle.
+}
+
+// EventLayerTreeDidChange [no description].
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#event-layerTreeDidChange
+type EventLayerTreeDidChange struct {
+	Layers []*Layer `json:"layers,omitempty"` // Layer tree, absent if not in the comspositing mode.
+}

vendor/github.com/chromedp/cdproto/layertree/layertree.go

@@ -0,0 +1,352 @@
+// Package layertree provides the Chrome DevTools Protocol
+// commands, types, and events for the LayerTree domain.
+//
+// Generated by the cdproto-gen command.
+package layertree
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"context"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/dom"
+	"github.com/mailru/easyjson"
+)
+
+// CompositingReasonsParams provides the reasons why the given layer was
+// composited.
+type CompositingReasonsParams struct {
+	LayerID LayerID `json:"layerId"` // The id of the layer for which we want to get the reasons it was composited.
+}
+
+// CompositingReasons provides the reasons why the given layer was
+// composited.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-compositingReasons
+//
+// parameters:
+//   layerID - The id of the layer for which we want to get the reasons it was composited.
+func CompositingReasons(layerID LayerID) *CompositingReasonsParams {
+	return &CompositingReasonsParams{
+		LayerID: layerID,
+	}
+}
+
+// CompositingReasonsReturns return values.
+type CompositingReasonsReturns struct {
+	CompositingReasonIds []string `json:"compositingReasonIds,omitempty"` // A list of strings specifying reason IDs for the given layer to become composited.
+}
+
+// Do executes LayerTree.compositingReasons against the provided context.
+//
+// returns:
+//   compositingReasonIds - A list of strings specifying reason IDs for the given layer to become composited.
+func (p *CompositingReasonsParams) Do(ctx context.Context) (compositingReasonIds []string, err error) {
+	// execute
+	var res CompositingReasonsReturns
+	err = cdp.Execute(ctx, CommandCompositingReasons, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.CompositingReasonIds, nil
+}
+
+// DisableParams disables compositing tree inspection.
+type DisableParams struct{}
+
+// Disable disables compositing tree inspection.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-disable
+func Disable() *DisableParams {
+	return &DisableParams{}
+}
+
+// Do executes LayerTree.disable against the provided context.
+func (p *DisableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandDisable, nil, nil)
+}
+
+// EnableParams enables compositing tree inspection.
+type EnableParams struct{}
+
+// Enable enables compositing tree inspection.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-enable
+func Enable() *EnableParams {
+	return &EnableParams{}
+}
+
+// Do executes LayerTree.enable against the provided context.
+func (p *EnableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandEnable, nil, nil)
+}
+
+// LoadSnapshotParams returns the snapshot identifier.
+type LoadSnapshotParams struct {
+	Tiles []*PictureTile `json:"tiles"` // An array of tiles composing the snapshot.
+}
+
+// LoadSnapshot returns the snapshot identifier.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-loadSnapshot
+//
+// parameters:
+//   tiles - An array of tiles composing the snapshot.
+func LoadSnapshot(tiles []*PictureTile) *LoadSnapshotParams {
+	return &LoadSnapshotParams{
+		Tiles: tiles,
+	}
+}
+
+// LoadSnapshotReturns return values.
+type LoadSnapshotReturns struct {
+	SnapshotID SnapshotID `json:"snapshotId,omitempty"` // The id of the snapshot.
+}
+
+// Do executes LayerTree.loadSnapshot against the provided context.
+//
+// returns:
+//   snapshotID - The id of the snapshot.
+func (p *LoadSnapshotParams) Do(ctx context.Context) (snapshotID SnapshotID, err error) {
+	// execute
+	var res LoadSnapshotReturns
+	err = cdp.Execute(ctx, CommandLoadSnapshot, p, &res)
+	if err != nil {
+		return "", err
+	}
+
+	return res.SnapshotID, nil
+}
+
+// MakeSnapshotParams returns the layer snapshot identifier.
+type MakeSnapshotParams struct {
+	LayerID LayerID `json:"layerId"` // The id of the layer.
+}
+
+// MakeSnapshot returns the layer snapshot identifier.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-makeSnapshot
+//
+// parameters:
+//   layerID - The id of the layer.
+func MakeSnapshot(layerID LayerID) *MakeSnapshotParams {
+	return &MakeSnapshotParams{
+		LayerID: layerID,
+	}
+}
+
+// MakeSnapshotReturns return values.
+type MakeSnapshotReturns struct {
+	SnapshotID SnapshotID `json:"snapshotId,omitempty"` // The id of the layer snapshot.
+}
+
+// Do executes LayerTree.makeSnapshot against the provided context.
+//
+// returns:
+//   snapshotID - The id of the layer snapshot.
+func (p *MakeSnapshotParams) Do(ctx context.Context) (snapshotID SnapshotID, err error) {
+	// execute
+	var res MakeSnapshotReturns
+	err = cdp.Execute(ctx, CommandMakeSnapshot, p, &res)
+	if err != nil {
+		return "", err
+	}
+
+	return res.SnapshotID, nil
+}
+
+// ProfileSnapshotParams [no description].
+type ProfileSnapshotParams struct {
+	SnapshotID     SnapshotID `json:"snapshotId"`               // The id of the layer snapshot.
+	MinRepeatCount int64      `json:"minRepeatCount,omitempty"` // The maximum number of times to replay the snapshot (1, if not specified).
+	MinDuration    float64    `json:"minDuration,omitempty"`    // The minimum duration (in seconds) to replay the snapshot.
+	ClipRect       *dom.Rect  `json:"clipRect,omitempty"`       // The clip rectangle to apply when replaying the snapshot.
+}
+
+// ProfileSnapshot [no description].
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-profileSnapshot
+//
+// parameters:
+//   snapshotID - The id of the layer snapshot.
+func ProfileSnapshot(snapshotID SnapshotID) *ProfileSnapshotParams {
+	return &ProfileSnapshotParams{
+		SnapshotID: snapshotID,
+	}
+}
+
+// WithMinRepeatCount the maximum number of times to replay the snapshot (1,
+// if not specified).
+func (p ProfileSnapshotParams) WithMinRepeatCount(minRepeatCount int64) *ProfileSnapshotParams {
+	p.MinRepeatCount = minRepeatCount
+	return &p
+}
+
+// WithMinDuration the minimum duration (in seconds) to replay the snapshot.
+func (p ProfileSnapshotParams) WithMinDuration(minDuration float64) *ProfileSnapshotParams {
+	p.MinDuration = minDuration
+	return &p
+}
+
+// WithClipRect the clip rectangle to apply when replaying the snapshot.
+func (p ProfileSnapshotParams) WithClipRect(clipRect *dom.Rect) *ProfileSnapshotParams {
+	p.ClipRect = clipRect
+	return &p
+}
+
+// ProfileSnapshotReturns return values.
+type ProfileSnapshotReturns struct {
+	Timings []PaintProfile `json:"timings,omitempty"` // The array of paint profiles, one per run.
+}
+
+// Do executes LayerTree.profileSnapshot against the provided context.
+//
+// returns:
+//   timings - The array of paint profiles, one per run.
+func (p *ProfileSnapshotParams) Do(ctx context.Context) (timings []PaintProfile, err error) {
+	// execute
+	var res ProfileSnapshotReturns
+	err = cdp.Execute(ctx, CommandProfileSnapshot, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Timings, nil
+}
+
+// ReleaseSnapshotParams releases layer snapshot captured by the back-end.
+type ReleaseSnapshotParams struct {
+	SnapshotID SnapshotID `json:"snapshotId"` // The id of the layer snapshot.
+}
+
+// ReleaseSnapshot releases layer snapshot captured by the back-end.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-releaseSnapshot
+//
+// parameters:
+//   snapshotID - The id of the layer snapshot.
+func ReleaseSnapshot(snapshotID SnapshotID) *ReleaseSnapshotParams {
+	return &ReleaseSnapshotParams{
+		SnapshotID: snapshotID,
+	}
+}
+
+// Do executes LayerTree.releaseSnapshot against the provided context.
+func (p *ReleaseSnapshotParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandReleaseSnapshot, p, nil)
+}
+
+// ReplaySnapshotParams replays the layer snapshot and returns the resulting
+// bitmap.
+type ReplaySnapshotParams struct {
+	SnapshotID SnapshotID `json:"snapshotId"`         // The id of the layer snapshot.
+	FromStep   int64      `json:"fromStep,omitempty"` // The first step to replay from (replay from the very start if not specified).
+	ToStep     int64      `json:"toStep,omitempty"`   // The last step to replay to (replay till the end if not specified).
+	Scale      float64    `json:"scale,omitempty"`    // The scale to apply while replaying (defaults to 1).
+}
+
+// ReplaySnapshot replays the layer snapshot and returns the resulting
+// bitmap.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-replaySnapshot
+//
+// parameters:
+//   snapshotID - The id of the layer snapshot.
+func ReplaySnapshot(snapshotID SnapshotID) *ReplaySnapshotParams {
+	return &ReplaySnapshotParams{
+		SnapshotID: snapshotID,
+	}
+}
+
+// WithFromStep the first step to replay from (replay from the very start if
+// not specified).
+func (p ReplaySnapshotParams) WithFromStep(fromStep int64) *ReplaySnapshotParams {
+	p.FromStep = fromStep
+	return &p
+}
+
+// WithToStep the last step to replay to (replay till the end if not
+// specified).
+func (p ReplaySnapshotParams) WithToStep(toStep int64) *ReplaySnapshotParams {
+	p.ToStep = toStep
+	return &p
+}
+
+// WithScale the scale to apply while replaying (defaults to 1).
+func (p ReplaySnapshotParams) WithScale(scale float64) *ReplaySnapshotParams {
+	p.Scale = scale
+	return &p
+}
+
+// ReplaySnapshotReturns return values.
+type ReplaySnapshotReturns struct {
+	DataURL string `json:"dataURL,omitempty"` // A data: URL for resulting image.
+}
+
+// Do executes LayerTree.replaySnapshot against the provided context.
+//
+// returns:
+//   dataURL - A data: URL for resulting image.
+func (p *ReplaySnapshotParams) Do(ctx context.Context) (dataURL string, err error) {
+	// execute
+	var res ReplaySnapshotReturns
+	err = cdp.Execute(ctx, CommandReplaySnapshot, p, &res)
+	if err != nil {
+		return "", err
+	}
+
+	return res.DataURL, nil
+}
+
+// SnapshotCommandLogParams replays the layer snapshot and returns canvas
+// log.
+type SnapshotCommandLogParams struct {
+	SnapshotID SnapshotID `json:"snapshotId"` // The id of the layer snapshot.
+}
+
+// SnapshotCommandLog replays the layer snapshot and returns canvas log.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#method-snapshotCommandLog
+//
+// parameters:
+//   snapshotID - The id of the layer snapshot.
+func SnapshotCommandLog(snapshotID SnapshotID) *SnapshotCommandLogParams {
+	return &SnapshotCommandLogParams{
+		SnapshotID: snapshotID,
+	}
+}
+
+// SnapshotCommandLogReturns return values.
+type SnapshotCommandLogReturns struct {
+	CommandLog []easyjson.RawMessage `json:"commandLog,omitempty"` // The array of canvas function calls.
+}
+
+// Do executes LayerTree.snapshotCommandLog against the provided context.
+//
+// returns:
+//   commandLog - The array of canvas function calls.
+func (p *SnapshotCommandLogParams) Do(ctx context.Context) (commandLog []easyjson.RawMessage, err error) {
+	// execute
+	var res SnapshotCommandLogReturns
+	err = cdp.Execute(ctx, CommandSnapshotCommandLog, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.CommandLog, nil
+}
+
+// Command names.
+const (
+	CommandCompositingReasons = "LayerTree.compositingReasons"
+	CommandDisable            = "LayerTree.disable"
+	CommandEnable             = "LayerTree.enable"
+	CommandLoadSnapshot       = "LayerTree.loadSnapshot"
+	CommandMakeSnapshot       = "LayerTree.makeSnapshot"
+	CommandProfileSnapshot    = "LayerTree.profileSnapshot"
+	CommandReleaseSnapshot    = "LayerTree.releaseSnapshot"
+	CommandReplaySnapshot     = "LayerTree.replaySnapshot"
+	CommandSnapshotCommandLog = "LayerTree.snapshotCommandLog"
+)

vendor/github.com/chromedp/cdproto/layertree/types.go

@@ -0,0 +1,135 @@
+package layertree
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"errors"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/dom"
+	"github.com/mailru/easyjson"
+	"github.com/mailru/easyjson/jlexer"
+	"github.com/mailru/easyjson/jwriter"
+)
+
+// LayerID unique Layer identifier.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-LayerId
+type LayerID string
+
+// String returns the LayerID as string value.
+func (t LayerID) String() string {
+	return string(t)
+}
+
+// SnapshotID unique snapshot identifier.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-SnapshotId
+type SnapshotID string
+
+// String returns the SnapshotID as string value.
+func (t SnapshotID) String() string {
+	return string(t)
+}
+
+// ScrollRect rectangle where scrolling happens on the main thread.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-ScrollRect
+type ScrollRect struct {
+	Rect *dom.Rect      `json:"rect"` // Rectangle itself.
+	Type ScrollRectType `json:"type"` // Reason for rectangle to force scrolling on the main thread
+}
+
+// StickyPositionConstraint sticky position constraints.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-StickyPositionConstraint
+type StickyPositionConstraint struct {
+	StickyBoxRect                       *dom.Rect `json:"stickyBoxRect"`                                 // Layout rectangle of the sticky element before being shifted
+	ContainingBlockRect                 *dom.Rect `json:"containingBlockRect"`                           // Layout rectangle of the containing block of the sticky element
+	NearestLayerShiftingStickyBox       LayerID   `json:"nearestLayerShiftingStickyBox,omitempty"`       // The nearest sticky layer that shifts the sticky box
+	NearestLayerShiftingContainingBlock LayerID   `json:"nearestLayerShiftingContainingBlock,omitempty"` // The nearest sticky layer that shifts the containing block
+}
+
+// PictureTile serialized fragment of layer picture along with its offset
+// within the layer.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-PictureTile
+type PictureTile struct {
+	X       float64 `json:"x"`       // Offset from owning layer left boundary
+	Y       float64 `json:"y"`       // Offset from owning layer top boundary
+	Picture string  `json:"picture"` // Base64-encoded snapshot data.
+}
+
+// Layer information about a compositing layer.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-Layer
+type Layer struct {
+	LayerID                  LayerID                   `json:"layerId"`                            // The unique id for this layer.
+	ParentLayerID            LayerID                   `json:"parentLayerId,omitempty"`            // The id of parent (not present for root).
+	BackendNodeID            cdp.BackendNodeID         `json:"backendNodeId,omitempty"`            // The backend id for the node associated with this layer.
+	OffsetX                  float64                   `json:"offsetX"`                            // Offset from parent layer, X coordinate.
+	OffsetY                  float64                   `json:"offsetY"`                            // Offset from parent layer, Y coordinate.
+	Width                    float64                   `json:"width"`                              // Layer width.
+	Height                   float64                   `json:"height"`                             // Layer height.
+	Transform                []float64                 `json:"transform,omitempty"`                // Transformation matrix for layer, default is identity matrix
+	AnchorX                  float64                   `json:"anchorX,omitempty"`                  // Transform anchor point X, absent if no transform specified
+	AnchorY                  float64                   `json:"anchorY,omitempty"`                  // Transform anchor point Y, absent if no transform specified
+	AnchorZ                  float64                   `json:"anchorZ,omitempty"`                  // Transform anchor point Z, absent if no transform specified
+	PaintCount               int64                     `json:"paintCount"`                         // Indicates how many time this layer has painted.
+	DrawsContent             bool                      `json:"drawsContent"`                       // Indicates whether this layer hosts any content, rather than being used for transform/scrolling purposes only.
+	Invisible                bool                      `json:"invisible,omitempty"`                // Set if layer is not visible.
+	ScrollRects              []*ScrollRect             `json:"scrollRects,omitempty"`              // Rectangles scrolling on main thread only.
+	StickyPositionConstraint *StickyPositionConstraint `json:"stickyPositionConstraint,omitempty"` // Sticky position constraint information
+}
+
+// PaintProfile array of timings, one per paint step.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-PaintProfile
+type PaintProfile []float64
+
+// ScrollRectType reason for rectangle to force scrolling on the main thread.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/LayerTree#type-ScrollRect
+type ScrollRectType string
+
+// String returns the ScrollRectType as string value.
+func (t ScrollRectType) String() string {
+	return string(t)
+}
+
+// ScrollRectType values.
+const (
+	ScrollRectTypeRepaintsOnScroll  ScrollRectType = "RepaintsOnScroll"
+	ScrollRectTypeTouchEventHandler ScrollRectType = "TouchEventHandler"
+	ScrollRectTypeWheelEventHandler ScrollRectType = "WheelEventHandler"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t ScrollRectType) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t ScrollRectType) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *ScrollRectType) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch ScrollRectType(in.String()) {
+	case ScrollRectTypeRepaintsOnScroll:
+		*t = ScrollRectTypeRepaintsOnScroll
+	case ScrollRectTypeTouchEventHandler:
+		*t = ScrollRectTypeTouchEventHandler
+	case ScrollRectTypeWheelEventHandler:
+		*t = ScrollRectTypeWheelEventHandler
+
+	default:
+		in.AddError(errors.New("unknown ScrollRectType value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *ScrollRectType) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}