Generic performer scrapers (#203)

* Generalise scraper API

* Add script performer scraper

* Fixes from testing

* Add context to scrapers and generalise

* Add scraping performer from URL

* Add error handling

* Move log to debug

* Add supported scrape types
This commit is contained in:
WithoutPants
2019-11-19 13:49:05 +11:00
committed by Leopere
parent 9bfa4e7560
commit 17247060b6
16 changed files with 836 additions and 132 deletions

View File

@@ -0,0 +1,67 @@
query ListScrapers($scraper_type: ScraperType!) {
listScrapers(scraper_type: $scraper_type) {
id
name
type
urls
supported_scrapes
}
}
query ScrapePerformerList($scraper_id: ID!, $query: String!) {
scrapePerformerList(scraper_id: $scraper_id, query: $query) {
name
url
birthdate
ethnicity
country
eye_color
height
measurements
fake_tits
career_length
tattoos
piercings
aliases
}
}
query ScrapePerformer($scraper_id: ID!, $scraped_performer: ScrapedPerformerInput!) {
scrapePerformer(scraper_id: $scraper_id, scraped_performer: $scraped_performer) {
name
url
twitter
instagram
birthdate
ethnicity
country
eye_color
height
measurements
fake_tits
career_length
tattoos
piercings
aliases
}
}
query ScrapePerformerURL($url: String!) {
scrapePerformerURL(url: $url) {
name
url
twitter
instagram
birthdate
ethnicity
country
eye_color
height
measurements
fake_tits
career_length
tattoos
piercings
aliases
}
}

View File

@@ -45,6 +45,15 @@ type Query {
# Scrapers
"""List available scrapers"""
listScrapers(scraper_type: ScraperType!): [Scraper!]!
"""Scrape a list of performers based on name"""
scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]!
"""Scrapes a complete performer record based on a scrapePerformerList result"""
scrapePerformer(scraper_id: ID!, scraped_performer: ScrapedPerformerInput!): ScrapedPerformer
"""Scrapes a complete performer record based on a URL"""
scrapePerformerURL(url: String!): ScrapedPerformer
"""Scrape a performer using Freeones"""
scrapeFreeones(performer_name: String!): ScrapedPerformer
"""Scrape a list of performers from a query"""

View File

@@ -15,4 +15,22 @@ type ScrapedPerformer {
tattoos: String
piercings: String
aliases: String
}
input ScrapedPerformerInput {
name: String
url: String
twitter: String
instagram: String
birthdate: String
ethnicity: String
country: String
eye_color: String
height: String
measurements: String
fake_tits: String
career_length: String
tattoos: String
piercings: String
aliases: String
}

View File

@@ -0,0 +1,16 @@
enum ScraperType {
PERFORMER
}
enum ScrapeType {
QUERY
URL
}
type Scraper {
id: ID!
name: String!
type: ScraperType!
urls: [String!]
supported_scrapes: [ScrapeType!]!
}