Add cdp support for xpath scrapers (#625)

Co-authored-by: WithoutPants <53250216+WithoutPants@users.noreply.github.com>

vendor/github.com/chromedp/cdproto/indexeddb/indexeddb.go

@@ -0,0 +1,325 @@
+// Package indexeddb provides the Chrome DevTools Protocol
+// commands, types, and events for the IndexedDB domain.
+//
+// Generated by the cdproto-gen command.
+package indexeddb
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"context"
+
+	"github.com/chromedp/cdproto/cdp"
+)
+
+// ClearObjectStoreParams clears all entries from an object store.
+type ClearObjectStoreParams struct {
+	SecurityOrigin  string `json:"securityOrigin"`  // Security origin.
+	DatabaseName    string `json:"databaseName"`    // Database name.
+	ObjectStoreName string `json:"objectStoreName"` // Object store name.
+}
+
+// ClearObjectStore clears all entries from an object store.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-clearObjectStore
+//
+// parameters:
+//   securityOrigin - Security origin.
+//   databaseName - Database name.
+//   objectStoreName - Object store name.
+func ClearObjectStore(securityOrigin string, databaseName string, objectStoreName string) *ClearObjectStoreParams {
+	return &ClearObjectStoreParams{
+		SecurityOrigin:  securityOrigin,
+		DatabaseName:    databaseName,
+		ObjectStoreName: objectStoreName,
+	}
+}
+
+// Do executes IndexedDB.clearObjectStore against the provided context.
+func (p *ClearObjectStoreParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandClearObjectStore, p, nil)
+}
+
+// DeleteDatabaseParams deletes a database.
+type DeleteDatabaseParams struct {
+	SecurityOrigin string `json:"securityOrigin"` // Security origin.
+	DatabaseName   string `json:"databaseName"`   // Database name.
+}
+
+// DeleteDatabase deletes a database.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-deleteDatabase
+//
+// parameters:
+//   securityOrigin - Security origin.
+//   databaseName - Database name.
+func DeleteDatabase(securityOrigin string, databaseName string) *DeleteDatabaseParams {
+	return &DeleteDatabaseParams{
+		SecurityOrigin: securityOrigin,
+		DatabaseName:   databaseName,
+	}
+}
+
+// Do executes IndexedDB.deleteDatabase against the provided context.
+func (p *DeleteDatabaseParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandDeleteDatabase, p, nil)
+}
+
+// DeleteObjectStoreEntriesParams delete a range of entries from an object
+// store.
+type DeleteObjectStoreEntriesParams struct {
+	SecurityOrigin  string    `json:"securityOrigin"`
+	DatabaseName    string    `json:"databaseName"`
+	ObjectStoreName string    `json:"objectStoreName"`
+	KeyRange        *KeyRange `json:"keyRange"` // Range of entry keys to delete
+}
+
+// DeleteObjectStoreEntries delete a range of entries from an object store.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-deleteObjectStoreEntries
+//
+// parameters:
+//   securityOrigin
+//   databaseName
+//   objectStoreName
+//   keyRange - Range of entry keys to delete
+func DeleteObjectStoreEntries(securityOrigin string, databaseName string, objectStoreName string, keyRange *KeyRange) *DeleteObjectStoreEntriesParams {
+	return &DeleteObjectStoreEntriesParams{
+		SecurityOrigin:  securityOrigin,
+		DatabaseName:    databaseName,
+		ObjectStoreName: objectStoreName,
+		KeyRange:        keyRange,
+	}
+}
+
+// Do executes IndexedDB.deleteObjectStoreEntries against the provided context.
+func (p *DeleteObjectStoreEntriesParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandDeleteObjectStoreEntries, p, nil)
+}
+
+// DisableParams disables events from backend.
+type DisableParams struct{}
+
+// Disable disables events from backend.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-disable
+func Disable() *DisableParams {
+	return &DisableParams{}
+}
+
+// Do executes IndexedDB.disable against the provided context.
+func (p *DisableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandDisable, nil, nil)
+}
+
+// EnableParams enables events from backend.
+type EnableParams struct{}
+
+// Enable enables events from backend.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-enable
+func Enable() *EnableParams {
+	return &EnableParams{}
+}
+
+// Do executes IndexedDB.enable against the provided context.
+func (p *EnableParams) Do(ctx context.Context) (err error) {
+	return cdp.Execute(ctx, CommandEnable, nil, nil)
+}
+
+// RequestDataParams requests data from object store or index.
+type RequestDataParams struct {
+	SecurityOrigin  string    `json:"securityOrigin"`     // Security origin.
+	DatabaseName    string    `json:"databaseName"`       // Database name.
+	ObjectStoreName string    `json:"objectStoreName"`    // Object store name.
+	IndexName       string    `json:"indexName"`          // Index name, empty string for object store data requests.
+	SkipCount       int64     `json:"skipCount"`          // Number of records to skip.
+	PageSize        int64     `json:"pageSize"`           // Number of records to fetch.
+	KeyRange        *KeyRange `json:"keyRange,omitempty"` // Key range.
+}
+
+// RequestData requests data from object store or index.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-requestData
+//
+// parameters:
+//   securityOrigin - Security origin.
+//   databaseName - Database name.
+//   objectStoreName - Object store name.
+//   indexName - Index name, empty string for object store data requests.
+//   skipCount - Number of records to skip.
+//   pageSize - Number of records to fetch.
+func RequestData(securityOrigin string, databaseName string, objectStoreName string, indexName string, skipCount int64, pageSize int64) *RequestDataParams {
+	return &RequestDataParams{
+		SecurityOrigin:  securityOrigin,
+		DatabaseName:    databaseName,
+		ObjectStoreName: objectStoreName,
+		IndexName:       indexName,
+		SkipCount:       skipCount,
+		PageSize:        pageSize,
+	}
+}
+
+// WithKeyRange key range.
+func (p RequestDataParams) WithKeyRange(keyRange *KeyRange) *RequestDataParams {
+	p.KeyRange = keyRange
+	return &p
+}
+
+// RequestDataReturns return values.
+type RequestDataReturns struct {
+	ObjectStoreDataEntries []*DataEntry `json:"objectStoreDataEntries,omitempty"` // Array of object store data entries.
+	HasMore                bool         `json:"hasMore,omitempty"`                // If true, there are more entries to fetch in the given range.
+}
+
+// Do executes IndexedDB.requestData against the provided context.
+//
+// returns:
+//   objectStoreDataEntries - Array of object store data entries.
+//   hasMore - If true, there are more entries to fetch in the given range.
+func (p *RequestDataParams) Do(ctx context.Context) (objectStoreDataEntries []*DataEntry, hasMore bool, err error) {
+	// execute
+	var res RequestDataReturns
+	err = cdp.Execute(ctx, CommandRequestData, p, &res)
+	if err != nil {
+		return nil, false, err
+	}
+
+	return res.ObjectStoreDataEntries, res.HasMore, nil
+}
+
+// GetMetadataParams gets metadata of an object store.
+type GetMetadataParams struct {
+	SecurityOrigin  string `json:"securityOrigin"`  // Security origin.
+	DatabaseName    string `json:"databaseName"`    // Database name.
+	ObjectStoreName string `json:"objectStoreName"` // Object store name.
+}
+
+// GetMetadata gets metadata of an object store.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-getMetadata
+//
+// parameters:
+//   securityOrigin - Security origin.
+//   databaseName - Database name.
+//   objectStoreName - Object store name.
+func GetMetadata(securityOrigin string, databaseName string, objectStoreName string) *GetMetadataParams {
+	return &GetMetadataParams{
+		SecurityOrigin:  securityOrigin,
+		DatabaseName:    databaseName,
+		ObjectStoreName: objectStoreName,
+	}
+}
+
+// GetMetadataReturns return values.
+type GetMetadataReturns struct {
+	EntriesCount      float64 `json:"entriesCount,omitempty"`      // the entries count
+	KeyGeneratorValue float64 `json:"keyGeneratorValue,omitempty"` // the current value of key generator, to become the next inserted key into the object store. Valid if objectStore.autoIncrement is true.
+}
+
+// Do executes IndexedDB.getMetadata against the provided context.
+//
+// returns:
+//   entriesCount - the entries count
+//   keyGeneratorValue - the current value of key generator, to become the next inserted key into the object store. Valid if objectStore.autoIncrement is true.
+func (p *GetMetadataParams) Do(ctx context.Context) (entriesCount float64, keyGeneratorValue float64, err error) {
+	// execute
+	var res GetMetadataReturns
+	err = cdp.Execute(ctx, CommandGetMetadata, p, &res)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	return res.EntriesCount, res.KeyGeneratorValue, nil
+}
+
+// RequestDatabaseParams requests database with given name in given frame.
+type RequestDatabaseParams struct {
+	SecurityOrigin string `json:"securityOrigin"` // Security origin.
+	DatabaseName   string `json:"databaseName"`   // Database name.
+}
+
+// RequestDatabase requests database with given name in given frame.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-requestDatabase
+//
+// parameters:
+//   securityOrigin - Security origin.
+//   databaseName - Database name.
+func RequestDatabase(securityOrigin string, databaseName string) *RequestDatabaseParams {
+	return &RequestDatabaseParams{
+		SecurityOrigin: securityOrigin,
+		DatabaseName:   databaseName,
+	}
+}
+
+// RequestDatabaseReturns return values.
+type RequestDatabaseReturns struct {
+	DatabaseWithObjectStores *DatabaseWithObjectStores `json:"databaseWithObjectStores,omitempty"` // Database with an array of object stores.
+}
+
+// Do executes IndexedDB.requestDatabase against the provided context.
+//
+// returns:
+//   databaseWithObjectStores - Database with an array of object stores.
+func (p *RequestDatabaseParams) Do(ctx context.Context) (databaseWithObjectStores *DatabaseWithObjectStores, err error) {
+	// execute
+	var res RequestDatabaseReturns
+	err = cdp.Execute(ctx, CommandRequestDatabase, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.DatabaseWithObjectStores, nil
+}
+
+// RequestDatabaseNamesParams requests database names for given security
+// origin.
+type RequestDatabaseNamesParams struct {
+	SecurityOrigin string `json:"securityOrigin"` // Security origin.
+}
+
+// RequestDatabaseNames requests database names for given security origin.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#method-requestDatabaseNames
+//
+// parameters:
+//   securityOrigin - Security origin.
+func RequestDatabaseNames(securityOrigin string) *RequestDatabaseNamesParams {
+	return &RequestDatabaseNamesParams{
+		SecurityOrigin: securityOrigin,
+	}
+}
+
+// RequestDatabaseNamesReturns return values.
+type RequestDatabaseNamesReturns struct {
+	DatabaseNames []string `json:"databaseNames,omitempty"` // Database names for origin.
+}
+
+// Do executes IndexedDB.requestDatabaseNames against the provided context.
+//
+// returns:
+//   databaseNames - Database names for origin.
+func (p *RequestDatabaseNamesParams) Do(ctx context.Context) (databaseNames []string, err error) {
+	// execute
+	var res RequestDatabaseNamesReturns
+	err = cdp.Execute(ctx, CommandRequestDatabaseNames, p, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	return res.DatabaseNames, nil
+}
+
+// Command names.
+const (
+	CommandClearObjectStore         = "IndexedDB.clearObjectStore"
+	CommandDeleteDatabase           = "IndexedDB.deleteDatabase"
+	CommandDeleteObjectStoreEntries = "IndexedDB.deleteObjectStoreEntries"
+	CommandDisable                  = "IndexedDB.disable"
+	CommandEnable                   = "IndexedDB.enable"
+	CommandRequestData              = "IndexedDB.requestData"
+	CommandGetMetadata              = "IndexedDB.getMetadata"
+	CommandRequestDatabase          = "IndexedDB.requestDatabase"
+	CommandRequestDatabaseNames     = "IndexedDB.requestDatabaseNames"
+)

vendor/github.com/chromedp/cdproto/indexeddb/types.go

@@ -0,0 +1,177 @@
+package indexeddb
+
+// Code generated by cdproto-gen. DO NOT EDIT.
+
+import (
+	"errors"
+
+	"github.com/chromedp/cdproto/runtime"
+	"github.com/mailru/easyjson"
+	"github.com/mailru/easyjson/jlexer"
+	"github.com/mailru/easyjson/jwriter"
+)
+
+// DatabaseWithObjectStores database with an array of object stores.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-DatabaseWithObjectStores
+type DatabaseWithObjectStores struct {
+	Name         string         `json:"name"`         // Database name.
+	Version      float64        `json:"version"`      // Database version (type is not 'integer', as the standard requires the version number to be 'unsigned long long')
+	ObjectStores []*ObjectStore `json:"objectStores"` // Object stores in this database.
+}
+
+// ObjectStore object store.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-ObjectStore
+type ObjectStore struct {
+	Name          string              `json:"name"`          // Object store name.
+	KeyPath       *KeyPath            `json:"keyPath"`       // Object store key path.
+	AutoIncrement bool                `json:"autoIncrement"` // If true, object store has auto increment flag set.
+	Indexes       []*ObjectStoreIndex `json:"indexes"`       // Indexes in this object store.
+}
+
+// ObjectStoreIndex object store index.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-ObjectStoreIndex
+type ObjectStoreIndex struct {
+	Name       string   `json:"name"`       // Index name.
+	KeyPath    *KeyPath `json:"keyPath"`    // Index key path.
+	Unique     bool     `json:"unique"`     // If true, index is unique.
+	MultiEntry bool     `json:"multiEntry"` // If true, index allows multiple entries for a key.
+}
+
+// Key Key.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-Key
+type Key struct {
+	Type   KeyType `json:"type"`             // Key type.
+	Number float64 `json:"number,omitempty"` // Number value.
+	String string  `json:"string,omitempty"` // String value.
+	Date   float64 `json:"date,omitempty"`   // Date value.
+	Array  []*Key  `json:"array,omitempty"`  // Array value.
+}
+
+// KeyRange key range.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-KeyRange
+type KeyRange struct {
+	Lower     *Key `json:"lower,omitempty"` // Lower bound.
+	Upper     *Key `json:"upper,omitempty"` // Upper bound.
+	LowerOpen bool `json:"lowerOpen"`       // If true lower bound is open.
+	UpperOpen bool `json:"upperOpen"`       // If true upper bound is open.
+}
+
+// DataEntry data entry.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-DataEntry
+type DataEntry struct {
+	Key        *runtime.RemoteObject `json:"key"`        // Key object.
+	PrimaryKey *runtime.RemoteObject `json:"primaryKey"` // Primary key object.
+	Value      *runtime.RemoteObject `json:"value"`      // Value object.
+}
+
+// KeyPath key path.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-KeyPath
+type KeyPath struct {
+	Type   KeyPathType `json:"type"`             // Key path type.
+	String string      `json:"string,omitempty"` // String value.
+	Array  []string    `json:"array,omitempty"`  // Array value.
+}
+
+// KeyType key type.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-Key
+type KeyType string
+
+// String returns the KeyType as string value.
+func (t KeyType) String() string {
+	return string(t)
+}
+
+// KeyType values.
+const (
+	KeyTypeNumber KeyType = "number"
+	KeyTypeString KeyType = "string"
+	KeyTypeDate   KeyType = "date"
+	KeyTypeArray  KeyType = "array"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t KeyType) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t KeyType) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *KeyType) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch KeyType(in.String()) {
+	case KeyTypeNumber:
+		*t = KeyTypeNumber
+	case KeyTypeString:
+		*t = KeyTypeString
+	case KeyTypeDate:
+		*t = KeyTypeDate
+	case KeyTypeArray:
+		*t = KeyTypeArray
+
+	default:
+		in.AddError(errors.New("unknown KeyType value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *KeyType) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}
+
+// KeyPathType key path type.
+//
+// See: https://chromedevtools.github.io/devtools-protocol/tot/IndexedDB#type-KeyPath
+type KeyPathType string
+
+// String returns the KeyPathType as string value.
+func (t KeyPathType) String() string {
+	return string(t)
+}
+
+// KeyPathType values.
+const (
+	KeyPathTypeNull   KeyPathType = "null"
+	KeyPathTypeString KeyPathType = "string"
+	KeyPathTypeArray  KeyPathType = "array"
+)
+
+// MarshalEasyJSON satisfies easyjson.Marshaler.
+func (t KeyPathType) MarshalEasyJSON(out *jwriter.Writer) {
+	out.String(string(t))
+}
+
+// MarshalJSON satisfies json.Marshaler.
+func (t KeyPathType) MarshalJSON() ([]byte, error) {
+	return easyjson.Marshal(t)
+}
+
+// UnmarshalEasyJSON satisfies easyjson.Unmarshaler.
+func (t *KeyPathType) UnmarshalEasyJSON(in *jlexer.Lexer) {
+	switch KeyPathType(in.String()) {
+	case KeyPathTypeNull:
+		*t = KeyPathTypeNull
+	case KeyPathTypeString:
+		*t = KeyPathTypeString
+	case KeyPathTypeArray:
+		*t = KeyPathTypeArray
+
+	default:
+		in.AddError(errors.New("unknown KeyPathType value"))
+	}
+}
+
+// UnmarshalJSON satisfies json.Unmarshaler.
+func (t *KeyPathType) UnmarshalJSON(buf []byte) error {
+	return easyjson.Unmarshal(buf, t)
+}