Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

vendor/github.com/chromedp/cdproto/storage/events.go

@@ -0,0 +1,36 @@
+package storage
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+// EventCacheStorageContentUpdated a cache's contents have been modified.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#event-cacheStorageContentUpdated
+type EventCacheStorageContentUpdated struct {
+	Origin    string `json:"origin"`    // Origin to update.
+	CacheName string `json:"cacheName"` // Name of cache in origin.
+}
+
+// EventCacheStorageListUpdated a cache has been added/deleted.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#event-cacheStorageListUpdated
+type EventCacheStorageListUpdated struct {
+	Origin string `json:"origin"` // Origin to update.
+}
+
+// EventIndexedDBContentUpdated the origin's IndexedDB object store has been
+// modified.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#event-indexedDBContentUpdated
+type EventIndexedDBContentUpdated struct {
+	Origin          string `json:"origin"`          // Origin to update.
+	DatabaseName    string `json:"databaseName"`    // Database to update.
+	ObjectStoreName string `json:"objectStoreName"` // ObjectStore to update.
+}
+
+// EventIndexedDBListUpdated the origin's IndexedDB database list has been
+// modified.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#event-indexedDBListUpdated
+type EventIndexedDBListUpdated struct {
+	Origin string `json:"origin"` // Origin to update.
+}

vendor/github.com/chromedp/cdproto/storage/storage.go

@@ -0,0 +1,286 @@
+// Package storage provides the Chrome DevTools Protocol
+// commands, types, and events for the Storage domain.
+//
+// Generated by the cdproto-gen command.
+package storage
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"context"
+
+	"github.com/chromedp/cdproto/cdp"
+	"github.com/chromedp/cdproto/network"
+)
+
+// ClearDataForOriginParams clears storage for origin.
+type ClearDataForOriginParams struct {
+	Origin       string `json:"origin"`       // Security origin.
+	StorageTypes string `json:"storageTypes"` // Comma separated list of StorageType to clear.
+}
+
+// ClearDataForOrigin clears storage for origin.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-clearDataForOrigin
+//
+// parameters:
+//   origin - Security origin.
+//   storageTypes - Comma separated list of StorageType to clear.
+func ClearDataForOrigin(origin string, storageTypes string) *ClearDataForOriginParams {
+	return &ClearDataForOriginParams{
+		Origin:       origin,
+		StorageTypes: storageTypes,
+	}
+}
+
+// Do executes Storage.clearDataForOrigin against the provided context.
+func (p *ClearDataForOriginParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandClearDataForOrigin, p, nil)
+}
+
+// GetCookiesParams returns all browser cookies.
+type GetCookiesParams struct {
+	BrowserContextID cdp.BrowserContextID `json:"browserContextId,omitempty"` // Browser context to use when called on the browser endpoint.
+}
+
+// GetCookies returns all browser cookies.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-getCookies
+//
+// parameters:
+func GetCookies() *GetCookiesParams {
+	return &GetCookiesParams{}
+}
+
+// WithBrowserContextID browser context to use when called on the browser
+// endpoint.
+func (p GetCookiesParams) WithBrowserContextID(browserContextID cdp.BrowserContextID) *GetCookiesParams {
+	p.BrowserContextID = browserContextID
+	return &p
+}
+
+// GetCookiesReturns return values.
+type GetCookiesReturns struct {
+	Cookies []*network.Cookie `json:"cookies,omitempty"` // Array of cookie objects.
+}
+
+// Do executes Storage.getCookies against the provided context.
+//
+// returns:
+//   cookies - Array of cookie objects.
+func (p *GetCookiesParams) Do(ctx context.Context) (cookies []*network.Cookie, err error) {
+	// execute
+	var res GetCookiesReturns
+	err = cdp.Execute(ctx, CommandGetCookies, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.Cookies, nil
+}
+
+// SetCookiesParams sets given cookies.
+type SetCookiesParams struct {
+	Cookies          []*network.CookieParam `json:"cookies"`                    // Cookies to be set.
+	BrowserContextID cdp.BrowserContextID   `json:"browserContextId,omitempty"` // Browser context to use when called on the browser endpoint.
+}
+
+// SetCookies sets given cookies.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-setCookies
+//
+// parameters:
+//   cookies - Cookies to be set.
+func SetCookies(cookies []*network.CookieParam) *SetCookiesParams {
+	return &SetCookiesParams{
+		Cookies: cookies,
+	}
+}
+
+// WithBrowserContextID browser context to use when called on the browser
+// endpoint.
+func (p SetCookiesParams) WithBrowserContextID(browserContextID cdp.BrowserContextID) *SetCookiesParams {
+	p.BrowserContextID = browserContextID
+	return &p
+}
+
+// Do executes Storage.setCookies against the provided context.
+func (p *SetCookiesParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandSetCookies, p, nil)
+}
+
+// ClearCookiesParams clears cookies.
+type ClearCookiesParams struct {
+	BrowserContextID cdp.BrowserContextID `json:"browserContextId,omitempty"` // Browser context to use when called on the browser endpoint.
+}
+
+// ClearCookies clears cookies.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-clearCookies
+//
+// parameters:
+func ClearCookies() *ClearCookiesParams {
+	return &ClearCookiesParams{}
+}
+
+// WithBrowserContextID browser context to use when called on the browser
+// endpoint.
+func (p ClearCookiesParams) WithBrowserContextID(browserContextID cdp.BrowserContextID) *ClearCookiesParams {
+	p.BrowserContextID = browserContextID
+	return &p
+}
+
+// Do executes Storage.clearCookies against the provided context.
+func (p *ClearCookiesParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandClearCookies, p, nil)
+}
+
+// GetUsageAndQuotaParams returns usage and quota in bytes.
+type GetUsageAndQuotaParams struct {
+	Origin string `json:"origin"` // Security origin.
+}
+
+// GetUsageAndQuota returns usage and quota in bytes.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-getUsageAndQuota
+//
+// parameters:
+//   origin - Security origin.
+func GetUsageAndQuota(origin string) *GetUsageAndQuotaParams {
+	return &GetUsageAndQuotaParams{
+		Origin: origin,
+	}
+}
+
+// GetUsageAndQuotaReturns return values.
+type GetUsageAndQuotaReturns struct {
+	Usage          float64         `json:"usage,omitempty"`          // Storage usage (bytes).
+	Quota          float64         `json:"quota,omitempty"`          // Storage quota (bytes).
+	UsageBreakdown []*UsageForType `json:"usageBreakdown,omitempty"` // Storage usage per type (bytes).
+}
+
+// Do executes Storage.getUsageAndQuota against the provided context.
+//
+// returns:
+//   usage - Storage usage (bytes).
+//   quota - Storage quota (bytes).
+//   usageBreakdown - Storage usage per type (bytes).
+func (p *GetUsageAndQuotaParams) Do(ctx context.Context) (usage float64, quota float64, usageBreakdown []*UsageForType, err error) {
+	// execute
+	var res GetUsageAndQuotaReturns
+	err = cdp.Execute(ctx, CommandGetUsageAndQuota, p, &res)
+	if err != nil {
+		return 0, 0, nil, err
+	}
+
+	return res.Usage, res.Quota, res.UsageBreakdown, nil
+}
+
+// TrackCacheStorageForOriginParams registers origin to be notified when an
+// update occurs to its cache storage list.
+type TrackCacheStorageForOriginParams struct {
+	Origin string `json:"origin"` // Security origin.
+}
+
+// TrackCacheStorageForOrigin registers origin to be notified when an update
+// occurs to its cache storage list.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-trackCacheStorageForOrigin
+//
+// parameters:
+//   origin - Security origin.
+func TrackCacheStorageForOrigin(origin string) *TrackCacheStorageForOriginParams {
+	return &TrackCacheStorageForOriginParams{
+		Origin: origin,
+	}
+}
+
+// Do executes Storage.trackCacheStorageForOrigin against the provided context.
+func (p *TrackCacheStorageForOriginParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandTrackCacheStorageForOrigin, p, nil)
+}
+
+// TrackIndexedDBForOriginParams registers origin to be notified when an
+// update occurs to its IndexedDB.
+type TrackIndexedDBForOriginParams struct {
+	Origin string `json:"origin"` // Security origin.
+}
+
+// TrackIndexedDBForOrigin registers origin to be notified when an update
+// occurs to its IndexedDB.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-trackIndexedDBForOrigin
+//
+// parameters:
+//   origin - Security origin.
+func TrackIndexedDBForOrigin(origin string) *TrackIndexedDBForOriginParams {
+	return &TrackIndexedDBForOriginParams{
+		Origin: origin,
+	}
+}
+
+// Do executes Storage.trackIndexedDBForOrigin against the provided context.
+func (p *TrackIndexedDBForOriginParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandTrackIndexedDBForOrigin, p, nil)
+}
+
+// UntrackCacheStorageForOriginParams unregisters origin from receiving
+// notifications for cache storage.
+type UntrackCacheStorageForOriginParams struct {
+	Origin string `json:"origin"` // Security origin.
+}
+
+// UntrackCacheStorageForOrigin unregisters origin from receiving
+// notifications for cache storage.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-untrackCacheStorageForOrigin
+//
+// parameters:
+//   origin - Security origin.
+func UntrackCacheStorageForOrigin(origin string) *UntrackCacheStorageForOriginParams {
+	return &UntrackCacheStorageForOriginParams{
+		Origin: origin,
+	}
+}
+
+// Do executes Storage.untrackCacheStorageForOrigin against the provided context.
+func (p *UntrackCacheStorageForOriginParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandUntrackCacheStorageForOrigin, p, nil)
+}
+
+// UntrackIndexedDBForOriginParams unregisters origin from receiving
+// notifications for IndexedDB.
+type UntrackIndexedDBForOriginParams struct {
+	Origin string `json:"origin"` // Security origin.
+}
+
+// UntrackIndexedDBForOrigin unregisters origin from receiving notifications
+// for IndexedDB.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#method-untrackIndexedDBForOrigin
+//
+// parameters:
+//   origin - Security origin.
+func UntrackIndexedDBForOrigin(origin string) *UntrackIndexedDBForOriginParams {
+	return &UntrackIndexedDBForOriginParams{
+		Origin: origin,
+	}
+}
+
+// Do executes Storage.untrackIndexedDBForOrigin against the provided context.
+func (p *UntrackIndexedDBForOriginParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandUntrackIndexedDBForOrigin, p, nil)
+}
+
+// Command names.
+const (
+	CommandClearDataForOrigin           = "Storage.clearDataForOrigin"
+	CommandGetCookies                   = "Storage.getCookies"
+	CommandSetCookies                   = "Storage.setCookies"
+	CommandClearCookies                 = "Storage.clearCookies"
+	CommandGetUsageAndQuota             = "Storage.getUsageAndQuota"
+	CommandTrackCacheStorageForOrigin   = "Storage.trackCacheStorageForOrigin"
+	CommandTrackIndexedDBForOrigin      = "Storage.trackIndexedDBForOrigin"
+	CommandUntrackCacheStorageForOrigin = "Storage.untrackCacheStorageForOrigin"
+	CommandUntrackIndexedDBForOrigin    = "Storage.untrackIndexedDBForOrigin"
+)

vendor/github.com/chromedp/cdproto/storage/types.go

@@ -0,0 +1,90 @@
+package storage
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"errors"
+
+	"github.com/mailru/easyjson"
+	"github.com/mailru/easyjson/jlexer"
+	"github.com/mailru/easyjson/jwriter"
+)
+
+// Type enum of possible storage types.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#type-StorageType
+type Type string
+
+// String returns the Type as string value.
+func (t Type) String() string {
+	return string(t)
+}
+
+// Type values.
+const (
+	TypeAppcache       Type = "appcache"
+	TypeCookies        Type = "cookies"
+	TypeFileSystems    Type = "file_systems"
+	TypeIndexeddb      Type = "indexeddb"
+	TypeLocalStorage   Type = "local_storage"
+	TypeShaderCache    Type = "shader_cache"
+	TypeWebsql         Type = "websql"
+	TypeServiceWorkers Type = "service_workers"
+	TypeCacheStorage   Type = "cache_storage"
+	TypeAll            Type = "all"
+	TypeOther          Type = "other"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t Type) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t Type) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *Type) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch Type(in.String()) {
+	case TypeAppcache:
+		*t = TypeAppcache
+	case TypeCookies:
+		*t = TypeCookies
+	case TypeFileSystems:
+		*t = TypeFileSystems
+	case TypeIndexeddb:
+		*t = TypeIndexeddb
+	case TypeLocalStorage:
+		*t = TypeLocalStorage
+	case TypeShaderCache:
+		*t = TypeShaderCache
+	case TypeWebsql:
+		*t = TypeWebsql
+	case TypeServiceWorkers:
+		*t = TypeServiceWorkers
+	case TypeCacheStorage:
+		*t = TypeCacheStorage
+	case TypeAll:
+		*t = TypeAll
+	case TypeOther:
+		*t = TypeOther
+
+	default:
+		in.AddError(errors.New("unknown Type value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *Type) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}
+
+// UsageForType usage for a storage type.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/Storage#type-UsageForType
+type UsageForType struct {
+	StorageType Type    `json:"storageType"` // Name of storage type.
+	Usage       float64 `json:"usage"`       // Storage usage (bytes).
+}