From 4625e1f95576d104196d8f00e4875fe8d5d02948 Mon Sep 17 00:00:00 2001 From: WithoutPants <53250216+WithoutPants@users.noreply.github.com> Date: Tue, 7 Sep 2021 11:54:22 +1000 Subject: [PATCH] Unify scrape refactor (#1630) * Unify scraped types * Make name fields optional * Unify single scrape queries * Change UI to use new interfaces * Add multi scrape interfaces * Use images instead of image --- gqlgen.yml | 16 -- graphql/documents/data/scrapers.graphql | 22 +- .../queries/scrapers/scrapers.graphql | 34 ++- graphql/schema/schema.graphql | 43 +++- graphql/schema/types/scraped-movie.graphql | 10 +- .../schema/types/scraped-performer.graphql | 10 +- graphql/schema/types/scraper.graphql | 135 ++++++---- pkg/api/resolver.go | 20 -- pkg/api/resolver_model_scraper.go | 23 -- pkg/api/resolver_query_scraper.go | 195 ++++++++++++++- pkg/manager/task_stash_box_tag.go | 18 +- pkg/models/model_scraped_item.go | 168 ------------- pkg/scraper/action.go | 6 +- pkg/scraper/config.go | 24 +- pkg/scraper/image.go | 2 + pkg/scraper/json.go | 32 +-- pkg/scraper/mapped.go | 20 +- pkg/scraper/matchers.go | 36 +-- pkg/scraper/scrapers.go | 110 ++++++--- pkg/scraper/script.go | 36 ++- pkg/scraper/stash.go | 232 ++++++++++-------- pkg/scraper/stashbox/stash_box.go | 161 ++++++++++-- pkg/scraper/xpath.go | 32 +-- pkg/scraper/xpath_test.go | 10 +- .../GalleryDetails/GalleryEditPanel.tsx | 11 +- .../GalleryDetails/GalleryScrapeDialog.tsx | 37 +-- .../Movies/MovieDetails/MovieEditPanel.tsx | 4 +- .../Movies/MovieDetails/MovieScrapeDialog.tsx | 9 +- .../PerformerDetails/PerformerEditPanel.tsx | 26 +- .../PerformerScrapeDialog.tsx | 20 +- .../PerformerDetails/PerformerScrapeModal.tsx | 2 +- .../PerformerStashBoxModal.tsx | 12 +- .../Scenes/SceneDetails/SceneEditPanel.tsx | 16 +- .../Scenes/SceneDetails/SceneScrapeDialog.tsx | 52 ++-- .../components/Tagger/StashSearchResult.tsx | 2 +- ui/v2.5/src/components/Tagger/TaggerList.tsx | 66 +---- ui/v2.5/src/components/Tagger/TaggerScene.tsx | 2 +- .../Tagger/performers/PerformerTagger.tsx | 6 +- ui/v2.5/src/components/Tagger/utils.ts | 6 +- ui/v2.5/src/core/StashService.ts | 150 ++++++----- 40 files changed, 1035 insertions(+), 781 deletions(-) delete mode 100644 pkg/api/resolver_model_scraper.go diff --git a/gqlgen.yml b/gqlgen.yml index eab0a4db9..8a21df01b 100644 --- a/gqlgen.yml +++ b/gqlgen.yml @@ -34,24 +34,8 @@ models: model: github.com/stashapp/stash/pkg/models.Movie Tag: model: github.com/stashapp/stash/pkg/models.Tag - ScrapedPerformer: - model: github.com/stashapp/stash/pkg/models.ScrapedPerformer - ScrapedScene: - model: github.com/stashapp/stash/pkg/models.ScrapedScene - ScrapedScenePerformer: - model: github.com/stashapp/stash/pkg/models.ScrapedScenePerformer - ScrapedSceneStudio: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneStudio - ScrapedSceneMovie: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneMovie - ScrapedSceneTag: - model: github.com/stashapp/stash/pkg/models.ScrapedSceneTag SceneFileType: model: github.com/stashapp/stash/pkg/models.SceneFileType - ScrapedMovie: - model: github.com/stashapp/stash/pkg/models.ScrapedMovie - ScrapedMovieStudio: - model: github.com/stashapp/stash/pkg/models.ScrapedMovieStudio SavedFilter: model: github.com/stashapp/stash/pkg/models.SavedFilter StashID: diff --git a/graphql/documents/data/scrapers.graphql b/graphql/documents/data/scrapers.graphql index b84fabfc9..7c4632b95 100644 --- a/graphql/documents/data/scrapers.graphql +++ b/graphql/documents/data/scrapers.graphql @@ -1,4 +1,5 @@ fragment ScrapedPerformerData on ScrapedPerformer { + stored_id name gender url @@ -18,7 +19,7 @@ fragment ScrapedPerformerData on ScrapedPerformer { tags { ...ScrapedSceneTagData } - image + images details death_date hair_color @@ -26,7 +27,7 @@ fragment ScrapedPerformerData on ScrapedPerformer { remote_site_id } -fragment ScrapedScenePerformerData on ScrapedScenePerformer { +fragment ScrapedScenePerformerData on ScrapedPerformer { stored_id name gender @@ -55,8 +56,8 @@ fragment ScrapedScenePerformerData on ScrapedScenePerformer { weight } -fragment ScrapedMovieStudioData on ScrapedMovieStudio { - id +fragment ScrapedMovieStudioData on ScrapedStudio { + stored_id name url } @@ -78,7 +79,7 @@ fragment ScrapedMovieData on ScrapedMovie { } } -fragment ScrapedSceneMovieData on ScrapedSceneMovie { +fragment ScrapedSceneMovieData on ScrapedMovie { stored_id name aliases @@ -90,14 +91,14 @@ fragment ScrapedSceneMovieData on ScrapedSceneMovie { synopsis } -fragment ScrapedSceneStudioData on ScrapedSceneStudio { +fragment ScrapedSceneStudioData on ScrapedStudio { stored_id name url remote_site_id } -fragment ScrapedSceneTagData on ScrapedSceneTag { +fragment ScrapedSceneTagData on ScrapedTag { stored_id name } @@ -108,6 +109,7 @@ fragment ScrapedSceneData on ScrapedScene { url date image + remote_site_id file { size @@ -135,6 +137,12 @@ fragment ScrapedSceneData on ScrapedScene { movies { ...ScrapedSceneMovieData } + + fingerprints { + hash + algorithm + duration + } } fragment ScrapedGalleryData on ScrapedGallery { diff --git a/graphql/documents/queries/scrapers/scrapers.graphql b/graphql/documents/queries/scrapers/scrapers.graphql index d5c54bac1..92c0bfd82 100644 --- a/graphql/documents/queries/scrapers/scrapers.graphql +++ b/graphql/documents/queries/scrapers/scrapers.graphql @@ -42,14 +42,14 @@ query ListMovieScrapers { } } -query ScrapePerformerList($scraper_id: ID!, $query: String!) { - scrapePerformerList(scraper_id: $scraper_id, query: $query) { +query ScrapeSinglePerformer($source: ScraperSourceInput!, $input: ScrapeSinglePerformerInput!) { + scrapeSinglePerformer(source: $source, input: $input) { ...ScrapedPerformerData } } -query ScrapePerformer($scraper_id: ID!, $scraped_performer: ScrapedPerformerInput!) { - scrapePerformer(scraper_id: $scraper_id, scraped_performer: $scraped_performer) { +query ScrapeMultiPerformers($source: ScraperSourceInput!, $input: ScrapeMultiPerformersInput!) { + scrapeMultiPerformers(source: $source, input: $input) { ...ScrapedPerformerData } } @@ -60,8 +60,14 @@ query ScrapePerformerURL($url: String!) { } } -query ScrapeScene($scraper_id: ID!, $scene: SceneUpdateInput!) { - scrapeScene(scraper_id: $scraper_id, scene: $scene) { +query ScrapeSingleScene($source: ScraperSourceInput!, $input: ScrapeSingleSceneInput!) { + scrapeSingleScene(source: $source, input: $input) { + ...ScrapedSceneData + } +} + +query ScrapeMultiScenes($source: ScraperSourceInput!, $input: ScrapeMultiScenesInput!) { + scrapeMultiScenes(source: $source, input: $input) { ...ScrapedSceneData } } @@ -72,8 +78,8 @@ query ScrapeSceneURL($url: String!) { } } -query ScrapeGallery($scraper_id: ID!, $gallery: GalleryUpdateInput!) { - scrapeGallery(scraper_id: $scraper_id, gallery: $gallery) { +query ScrapeSingleGallery($source: ScraperSourceInput!, $input: ScrapeSingleGalleryInput!) { + scrapeSingleGallery(source: $source, input: $input) { ...ScrapedGalleryData } } @@ -89,15 +95,3 @@ query ScrapeMovieURL($url: String!) { ...ScrapedMovieData } } - -query QueryStashBoxScene($input: StashBoxSceneQueryInput!) { - queryStashBoxScene(input: $input) { - ...ScrapedStashBoxSceneData - } -} - -query QueryStashBoxPerformer($input: StashBoxPerformerQueryInput!) { - queryStashBoxPerformer(input: $input) { - ...ScrapedStashBoxPerformerData - } -} diff --git a/graphql/schema/schema.graphql b/graphql/schema/schema.graphql index c0ba269ef..64a55c7eb 100644 --- a/graphql/schema/schema.graphql +++ b/graphql/schema/schema.graphql @@ -72,31 +72,50 @@ type Query { listGalleryScrapers: [Scraper!]! listMovieScrapers: [Scraper!]! - """Scrape a list of performers based on name""" - scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]! - """Scrapes a complete performer record based on a scrapePerformerList result""" - scrapePerformer(scraper_id: ID!, scraped_performer: ScrapedPerformerInput!): ScrapedPerformer + """Scrape for a single scene""" + scrapeSingleScene(source: ScraperSourceInput!, input: ScrapeSingleSceneInput!): [ScrapedScene!]! + """Scrape for multiple scenes""" + scrapeMultiScenes(source: ScraperSourceInput!, input: ScrapeMultiScenesInput!): [[ScrapedScene!]!]! + + """Scrape for a single performer""" + scrapeSinglePerformer(source: ScraperSourceInput!, input: ScrapeSinglePerformerInput!): [ScrapedPerformer!]! + """Scrape for multiple performers""" + scrapeMultiPerformers(source: ScraperSourceInput!, input: ScrapeMultiPerformersInput!): [[ScrapedPerformer!]!]! + + """Scrape for a single gallery""" + scrapeSingleGallery(source: ScraperSourceInput!, input: ScrapeSingleGalleryInput!): [ScrapedGallery!]! + + """Scrape for a single movie""" + scrapeSingleMovie(source: ScraperSourceInput!, input: ScrapeSingleMovieInput!): [ScrapedMovie!]! + """Scrapes a complete performer record based on a URL""" scrapePerformerURL(url: String!): ScrapedPerformer - """Scrapes a complete scene record based on an existing scene""" - scrapeScene(scraper_id: ID!, scene: SceneUpdateInput!): ScrapedScene """Scrapes a complete performer record based on a URL""" scrapeSceneURL(url: String!): ScrapedScene - """Scrapes a complete gallery record based on an existing gallery""" - scrapeGallery(scraper_id: ID!, gallery: GalleryUpdateInput!): ScrapedGallery """Scrapes a complete gallery record based on a URL""" scrapeGalleryURL(url: String!): ScrapedGallery """Scrapes a complete movie record based on a URL""" scrapeMovieURL(url: String!): ScrapedMovie + """Scrape a list of performers based on name""" + scrapePerformerList(scraper_id: ID!, query: String!): [ScrapedPerformer!]! @deprecated(reason: "use scrapeSinglePerformer") + """Scrapes a complete performer record based on a scrapePerformerList result""" + scrapePerformer(scraper_id: ID!, scraped_performer: ScrapedPerformerInput!): ScrapedPerformer @deprecated(reason: "use scrapeSinglePerformer") + """Scrapes a complete scene record based on an existing scene""" + scrapeScene(scraper_id: ID!, scene: SceneUpdateInput!): ScrapedScene @deprecated(reason: "use scrapeSingleScene") + """Scrapes a complete gallery record based on an existing gallery""" + scrapeGallery(scraper_id: ID!, gallery: GalleryUpdateInput!): ScrapedGallery @deprecated(reason: "use scrapeSingleGallery") + """Scrape a performer using Freeones""" - scrapeFreeones(performer_name: String!): ScrapedPerformer + scrapeFreeones(performer_name: String!): ScrapedPerformer @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones") """Scrape a list of performers from a query""" - scrapeFreeonesPerformerList(query: String!): [String!]! + scrapeFreeonesPerformerList(query: String!): [String!]! @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones") """Query StashBox for scenes""" - queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! - queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! + queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! @deprecated(reason: "use scrapeSingleScene or scrapeMultiScenes") + """Query StashBox for performers""" + queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! @deprecated(reason: "use scrapeSinglePerformer or scrapeMultiPerformers") + # === end deprecated methods === # Plugins """List loaded plugins""" diff --git a/graphql/schema/types/scraped-movie.graphql b/graphql/schema/types/scraped-movie.graphql index d1546dfb9..55efb693d 100644 --- a/graphql/schema/types/scraped-movie.graphql +++ b/graphql/schema/types/scraped-movie.graphql @@ -1,12 +1,6 @@ -type ScrapedMovieStudio { - """Set if studio matched""" - id: ID - name: String! - url: String -} - """A movie from a scraping operation...""" type ScrapedMovie { + stored_id: ID name: String aliases: String duration: String @@ -15,7 +9,7 @@ type ScrapedMovie { director: String url: String synopsis: String - studio: ScrapedMovieStudio + studio: ScrapedStudio """This should be a base64 encoded data URL""" front_image: String diff --git a/graphql/schema/types/scraped-performer.graphql b/graphql/schema/types/scraped-performer.graphql index 2ae1b5a8a..b11b9b1b5 100644 --- a/graphql/schema/types/scraped-performer.graphql +++ b/graphql/schema/types/scraped-performer.graphql @@ -1,5 +1,7 @@ """A performer from a scraping operation...""" type ScrapedPerformer { + """Set if performer matched""" + stored_id: ID name: String gender: String url: String @@ -16,11 +18,11 @@ type ScrapedPerformer { tattoos: String piercings: String aliases: String - # Should be ScrapedPerformerTag - but would be identical types - tags: [ScrapedSceneTag!] + tags: [ScrapedTag!] """This should be a base64 encoded data URL""" - image: String + image: String @deprecated(reason: "use images instead") + images: [String!] details: String death_date: String hair_color: String @@ -29,6 +31,8 @@ type ScrapedPerformer { } input ScrapedPerformerInput { + """Set if performer matched""" + stored_id: ID name: String gender: String url: String diff --git a/graphql/schema/types/scraper.graphql b/graphql/schema/types/scraper.graphql index 860457bb0..9e35346f4 100644 --- a/graphql/schema/types/scraper.graphql +++ b/graphql/schema/types/scraper.graphql @@ -26,49 +26,7 @@ type Scraper { movie: ScraperSpec } -type ScrapedScenePerformer { - """Set if performer matched""" - stored_id: ID - name: String! - gender: String - url: String - twitter: String - instagram: String - birthdate: String - ethnicity: String - country: String - eye_color: String - height: String - measurements: String - fake_tits: String - career_length: String - tattoos: String - piercings: String - aliases: String - tags: [ScrapedSceneTag!] - - remote_site_id: String - images: [String!] - details: String - death_date: String - hair_color: String - weight: String -} - -type ScrapedSceneMovie { - """Set if movie matched""" - stored_id: ID - name: String! - aliases: String - duration: String - date: String - rating: String - director: String - synopsis: String - url: String -} - -type ScrapedSceneStudio { +type ScrapedStudio { """Set if studio matched""" stored_id: ID name: String! @@ -77,7 +35,7 @@ type ScrapedSceneStudio { remote_site_id: String } -type ScrapedSceneTag { +type ScrapedTag { """Set if tag matched""" stored_id: ID name: String! @@ -94,25 +52,98 @@ type ScrapedScene { file: SceneFileType # Resolver - studio: ScrapedSceneStudio - tags: [ScrapedSceneTag!] - performers: [ScrapedScenePerformer!] - movies: [ScrapedSceneMovie!] + studio: ScrapedStudio + tags: [ScrapedTag!] + performers: [ScrapedPerformer!] + movies: [ScrapedMovie!] remote_site_id: String duration: Int fingerprints: [StashBoxFingerprint!] } +input ScrapedSceneInput { + title: String + details: String + url: String + date: String + + # no image, file, duration or relationships + + remote_site_id: String +} + type ScrapedGallery { title: String details: String url: String date: String - studio: ScrapedSceneStudio - tags: [ScrapedSceneTag!] - performers: [ScrapedScenePerformer!] + studio: ScrapedStudio + tags: [ScrapedTag!] + performers: [ScrapedPerformer!] +} + +input ScrapedGalleryInput { + title: String + details: String + url: String + date: String + + # no studio, tags or performers +} + +input ScraperSourceInput { + """Index of the configured stash-box instance to use. Should be unset if scraper_id is set""" + stash_box_index: Int + """Scraper ID to scrape with. Should be unset if stash_box_index is set""" + scraper_id: ID +} + +input ScrapeSingleSceneInput { + """Instructs to query by string""" + query: String + """Instructs to query by scene fingerprints""" + scene_id: ID + """Instructs to query by scene fragment""" + scene_input: ScrapedSceneInput +} + +input ScrapeMultiScenesInput { + """Instructs to query by scene fingerprints""" + scene_ids: [ID!] +} + +input ScrapeSinglePerformerInput { + """Instructs to query by string""" + query: String + """Instructs to query by performer id""" + performer_id: ID + """Instructs to query by performer fragment""" + performer_input: ScrapedPerformerInput +} + +input ScrapeMultiPerformersInput { + """Instructs to query by scene fingerprints""" + performer_ids: [ID!] +} + +input ScrapeSingleGalleryInput { + """Instructs to query by string""" + query: String + """Instructs to query by gallery id""" + gallery_id: ID + """Instructs to query by gallery fragment""" + gallery_input: ScrapedGalleryInput +} + +input ScrapeSingleMovieInput { + """Instructs to query by string""" + query: String + """Instructs to query by movie id""" + movie_id: ID + """Instructs to query by gallery fragment""" + movie_input: ScrapedMovieInput } input StashBoxSceneQueryInput { @@ -135,7 +166,7 @@ input StashBoxPerformerQueryInput { type StashBoxPerformerQueryResult { query: String! - results: [ScrapedScenePerformer!]! + results: [ScrapedPerformer!]! } type StashBoxFingerprint { diff --git a/pkg/api/resolver.go b/pkg/api/resolver.go index ed15a1d21..7d3f6aa3f 100644 --- a/pkg/api/resolver.go +++ b/pkg/api/resolver.go @@ -53,22 +53,6 @@ func (r *Resolver) Tag() models.TagResolver { return &tagResolver{r} } -func (r *Resolver) ScrapedSceneTag() models.ScrapedSceneTagResolver { - return &scrapedSceneTagResolver{r} -} - -func (r *Resolver) ScrapedSceneMovie() models.ScrapedSceneMovieResolver { - return &scrapedSceneMovieResolver{r} -} - -func (r *Resolver) ScrapedScenePerformer() models.ScrapedScenePerformerResolver { - return &scrapedScenePerformerResolver{r} -} - -func (r *Resolver) ScrapedSceneStudio() models.ScrapedSceneStudioResolver { - return &scrapedSceneStudioResolver{r} -} - type mutationResolver struct{ *Resolver } type queryResolver struct{ *Resolver } type subscriptionResolver struct{ *Resolver } @@ -81,10 +65,6 @@ type imageResolver struct{ *Resolver } type studioResolver struct{ *Resolver } type movieResolver struct{ *Resolver } type tagResolver struct{ *Resolver } -type scrapedSceneTagResolver struct{ *Resolver } -type scrapedSceneMovieResolver struct{ *Resolver } -type scrapedScenePerformerResolver struct{ *Resolver } -type scrapedSceneStudioResolver struct{ *Resolver } func (r *Resolver) withTxn(ctx context.Context, fn func(r models.Repository) error) error { return r.txnManager.WithTxn(ctx, fn) diff --git a/pkg/api/resolver_model_scraper.go b/pkg/api/resolver_model_scraper.go deleted file mode 100644 index 583194496..000000000 --- a/pkg/api/resolver_model_scraper.go +++ /dev/null @@ -1,23 +0,0 @@ -package api - -import ( - "context" - - "github.com/stashapp/stash/pkg/models" -) - -func (r *scrapedSceneTagResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneTag) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedSceneMovieResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneMovie) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedScenePerformerResolver) StoredID(ctx context.Context, obj *models.ScrapedScenePerformer) (*string, error) { - return obj.ID, nil -} - -func (r *scrapedSceneStudioResolver) StoredID(ctx context.Context, obj *models.ScrapedSceneStudio) (*string, error) { - return obj.ID, nil -} diff --git a/pkg/api/resolver_query_scraper.go b/pkg/api/resolver_query_scraper.go index 301870351..55b17c09f 100644 --- a/pkg/api/resolver_query_scraper.go +++ b/pkg/api/resolver_query_scraper.go @@ -2,7 +2,9 @@ package api import ( "context" + "errors" "fmt" + "strconv" "github.com/stashapp/stash/pkg/manager" "github.com/stashapp/stash/pkg/manager/config" @@ -29,8 +31,9 @@ func (r *queryResolver) ScrapeFreeonesPerformerList(ctx context.Context, query s var ret []string for _, v := range scrapedPerformers { - name := v.Name - ret = append(ret, *name) + if v.Name != nil { + ret = append(ret, *v.Name) + } } return ret, nil @@ -69,7 +72,12 @@ func (r *queryResolver) ScrapePerformerURL(ctx context.Context, url string) (*mo } func (r *queryResolver) ScrapeScene(ctx context.Context, scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - return manager.GetInstance().ScraperCache.ScrapeScene(scraperID, scene) + id, err := strconv.Atoi(scene.ID) + if err != nil { + return nil, err + } + + return manager.GetInstance().ScraperCache.ScrapeScene(scraperID, id) } func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models.ScrapedScene, error) { @@ -77,7 +85,12 @@ func (r *queryResolver) ScrapeSceneURL(ctx context.Context, url string) (*models } func (r *queryResolver) ScrapeGallery(ctx context.Context, scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - return manager.GetInstance().ScraperCache.ScrapeGallery(scraperID, gallery) + id, err := strconv.Atoi(gallery.ID) + if err != nil { + return nil, err + } + + return manager.GetInstance().ScraperCache.ScrapeGallery(scraperID, id) } func (r *queryResolver) ScrapeGalleryURL(ctx context.Context, url string) (*models.ScrapedGallery, error) { @@ -98,7 +111,7 @@ func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.Sta client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager) if len(input.SceneIds) > 0 { - return client.FindStashBoxScenesByFingerprints(input.SceneIds) + return client.FindStashBoxScenesByFingerprintsFlat(input.SceneIds) } if input.Q != nil { @@ -127,3 +140,175 @@ func (r *queryResolver) QueryStashBoxPerformer(ctx context.Context, input models return nil, nil } + +func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) { + boxes := config.GetInstance().GetStashBoxes() + + if index < 0 || index >= len(boxes) { + return nil, fmt.Errorf("invalid stash_box_index %d", index) + } + + return stashbox.NewClient(*boxes[index], r.txnManager), nil +} + +func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) { + if source.ScraperID != nil { + var singleScene *models.ScrapedScene + var err error + + if input.SceneID != nil { + var sceneID int + sceneID, err = strconv.Atoi(*input.SceneID) + if err != nil { + return nil, err + } + singleScene, err = manager.GetInstance().ScraperCache.ScrapeScene(*source.ScraperID, sceneID) + } else if input.SceneInput != nil { + singleScene, err = manager.GetInstance().ScraperCache.ScrapeSceneFragment(*source.ScraperID, *input.SceneInput) + } else { + return nil, errors.New("not implemented") + } + + if err != nil { + return nil, err + } + + if singleScene != nil { + return []*models.ScrapedScene{singleScene}, nil + } + + return nil, nil + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + if input.SceneID != nil { + return client.FindStashBoxScenesByFingerprintsFlat([]string{*input.SceneID}) + } else if input.Query != nil { + return client.QueryStashBoxScene(*input.Query) + } + + return nil, errors.New("scene_id or query must be set") + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiScenesInput) ([][]*models.ScrapedScene, error) { + if source.ScraperID != nil { + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + return client.FindStashBoxScenesByFingerprints(input.SceneIds) + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeSinglePerformer(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSinglePerformerInput) ([]*models.ScrapedPerformer, error) { + if source.ScraperID != nil { + if input.PerformerInput != nil { + singlePerformer, err := manager.GetInstance().ScraperCache.ScrapePerformer(*source.ScraperID, *input.PerformerInput) + if err != nil { + return nil, err + } + + if singlePerformer != nil { + return []*models.ScrapedPerformer{singlePerformer}, nil + } + + return nil, nil + } + + if input.Query != nil { + return manager.GetInstance().ScraperCache.ScrapePerformerList(*source.ScraperID, *input.Query) + } + + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + var ret []*models.StashBoxPerformerQueryResult + if input.PerformerID != nil { + ret, err = client.FindStashBoxPerformersByNames([]string{*input.PerformerID}) + } else if input.Query != nil { + ret, err = client.QueryStashBoxPerformer(*input.Query) + } else { + return nil, errors.New("not implemented") + } + + if err != nil { + return nil, err + } + + if len(ret) > 0 { + return ret[0].Results, nil + } + + return nil, nil + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeMultiPerformers(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeMultiPerformersInput) ([][]*models.ScrapedPerformer, error) { + if source.ScraperID != nil { + return nil, errors.New("not implemented") + } else if source.StashBoxIndex != nil { + client, err := r.getStashBoxClient(*source.StashBoxIndex) + if err != nil { + return nil, err + } + + return client.FindStashBoxPerformersByPerformerNames(input.PerformerIds) + } + + return nil, errors.New("scraper_id or stash_box_index must be set") +} + +func (r *queryResolver) ScrapeSingleGallery(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleGalleryInput) ([]*models.ScrapedGallery, error) { + if source.ScraperID != nil { + var singleGallery *models.ScrapedGallery + var err error + + if input.GalleryID != nil { + var galleryID int + galleryID, err = strconv.Atoi(*input.GalleryID) + if err != nil { + return nil, err + } + singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGallery(*source.ScraperID, galleryID) + } else if input.GalleryInput != nil { + singleGallery, err = manager.GetInstance().ScraperCache.ScrapeGalleryFragment(*source.ScraperID, *input.GalleryInput) + } else { + return nil, errors.New("not implemented") + } + + if err != nil { + return nil, err + } + + if singleGallery != nil { + return []*models.ScrapedGallery{singleGallery}, nil + } + + return nil, nil + } else if source.StashBoxIndex != nil { + return nil, errors.New("not supported") + } + + return nil, errors.New("scraper_id must be set") +} + +func (r *queryResolver) ScrapeSingleMovie(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleMovieInput) ([]*models.ScrapedMovie, error) { + return nil, errors.New("not supported") +} diff --git a/pkg/manager/task_stash_box_tag.go b/pkg/manager/task_stash_box_tag.go index 597655e9f..5bd5c2252 100644 --- a/pkg/manager/task_stash_box_tag.go +++ b/pkg/manager/task_stash_box_tag.go @@ -40,7 +40,7 @@ func (t *StashBoxPerformerTagTask) Description() string { } func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { - var performer *models.ScrapedScenePerformer + var performer *models.ScrapedPerformer var err error client := stashbox.NewClient(*t.box, t.txnManager) @@ -132,8 +132,8 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { value := getNullString(performer.Measurements) partial.Measurements = &value } - if excluded["name"] { - value := sql.NullString{String: performer.Name, Valid: true} + if excluded["name"] && performer.Name != nil { + value := sql.NullString{String: *performer.Name, Valid: true} partial.Name = &value } if performer.Piercings != nil && !excluded["piercings"] { @@ -180,17 +180,21 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { } if err == nil { - logger.Infof("Updated performer %s", performer.Name) + var name string + if performer.Name != nil { + name = *performer.Name + } + logger.Infof("Updated performer %s", name) } return err }) - } else if t.name != nil { + } else if t.name != nil && performer.Name != nil { currentTime := time.Now() newPerformer := models.Performer{ Aliases: getNullString(performer.Aliases), Birthdate: getDate(performer.Birthdate), CareerLength: getNullString(performer.CareerLength), - Checksum: utils.MD5FromString(performer.Name), + Checksum: utils.MD5FromString(*performer.Name), Country: getNullString(performer.Country), CreatedAt: models.SQLiteTimestamp{Timestamp: currentTime}, Ethnicity: getNullString(performer.Ethnicity), @@ -201,7 +205,7 @@ func (t *StashBoxPerformerTagTask) stashBoxPerformerTag() { Height: getNullString(performer.Height), Instagram: getNullString(performer.Instagram), Measurements: getNullString(performer.Measurements), - Name: sql.NullString{String: performer.Name, Valid: true}, + Name: sql.NullString{String: *performer.Name, Valid: true}, Piercings: getNullString(performer.Piercings), Tattoos: getNullString(performer.Tattoos), Twitter: getNullString(performer.Twitter), diff --git a/pkg/models/model_scraped_item.go b/pkg/models/model_scraped_item.go index 230fd0ba0..4035163b7 100644 --- a/pkg/models/model_scraped_item.go +++ b/pkg/models/model_scraped_item.go @@ -23,174 +23,6 @@ type ScrapedItem struct { UpdatedAt SQLiteTimestamp `db:"updated_at" json:"updated_at"` } -type ScrapedPerformer struct { - Name *string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Image *string `graphql:"image" json:"image"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` -} - -// this type has no Image field -type ScrapedPerformerStash struct { - Name *string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` -} - -type ScrapedScene struct { - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - Image *string `graphql:"image" json:"image"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` - Duration *int `graphql:"duration" json:"duration"` - File *SceneFileType `graphql:"file" json:"file"` - Fingerprints []*StashBoxFingerprint `graphql:"fingerprints" json:"fingerprints"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Movies []*ScrapedSceneMovie `graphql:"movies" json:"movies"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -// stash doesn't return image, and we need id -type ScrapedSceneStash struct { - ID string `graphql:"id" json:"id"` - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - File *SceneFileType `graphql:"file" json:"file"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -type ScrapedGalleryStash struct { - ID string `graphql:"id" json:"id"` - Title *string `graphql:"title" json:"title"` - Details *string `graphql:"details" json:"details"` - URL *string `graphql:"url" json:"url"` - Date *string `graphql:"date" json:"date"` - File *SceneFileType `graphql:"file" json:"file"` - Studio *ScrapedSceneStudio `graphql:"studio" json:"studio"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - Performers []*ScrapedScenePerformer `graphql:"performers" json:"performers"` -} - -type ScrapedScenePerformer struct { - // Set if performer matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - Gender *string `graphql:"gender" json:"gender"` - URL *string `graphql:"url" json:"url"` - Twitter *string `graphql:"twitter" json:"twitter"` - Instagram *string `graphql:"instagram" json:"instagram"` - Birthdate *string `graphql:"birthdate" json:"birthdate"` - Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` - Country *string `graphql:"country" json:"country"` - EyeColor *string `graphql:"eye_color" json:"eye_color"` - Height *string `graphql:"height" json:"height"` - Measurements *string `graphql:"measurements" json:"measurements"` - FakeTits *string `graphql:"fake_tits" json:"fake_tits"` - CareerLength *string `graphql:"career_length" json:"career_length"` - Tattoos *string `graphql:"tattoos" json:"tattoos"` - Piercings *string `graphql:"piercings" json:"piercings"` - Aliases *string `graphql:"aliases" json:"aliases"` - Tags []*ScrapedSceneTag `graphql:"tags" json:"tags"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` - Images []string `graphql:"images" json:"images"` - Details *string `graphql:"details" json:"details"` - DeathDate *string `graphql:"death_date" json:"death_date"` - HairColor *string `graphql:"hair_color" json:"hair_color"` - Weight *string `graphql:"weight" json:"weight"` -} - -type ScrapedSceneStudio struct { - // Set if studio matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - URL *string `graphql:"url" json:"url"` - RemoteSiteID *string `graphql:"remote_site_id" json:"remote_site_id"` -} - -type ScrapedSceneMovie struct { - // Set if movie matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - Aliases string `graphql:"aliases" json:"aliases"` - Duration string `graphql:"duration" json:"duration"` - Date string `graphql:"date" json:"date"` - Rating string `graphql:"rating" json:"rating"` - Director string `graphql:"director" json:"director"` - Synopsis string `graphql:"synopsis" json:"synopsis"` - URL *string `graphql:"url" json:"url"` -} - -type ScrapedSceneTag struct { - // Set if tag matched - ID *string `graphql:"stored_id" json:"stored_id"` - Name string `graphql:"name" json:"name"` -} - -type ScrapedMovie struct { - Name *string `graphql:"name" json:"name"` - Aliases *string `graphql:"aliases" json:"aliases"` - Duration *string `graphql:"duration" json:"duration"` - Date *string `graphql:"date" json:"date"` - Rating *string `graphql:"rating" json:"rating"` - Director *string `graphql:"director" json:"director"` - Studio *ScrapedMovieStudio `graphql:"studio" json:"studio"` - Synopsis *string `graphql:"synopsis" json:"synopsis"` - URL *string `graphql:"url" json:"url"` - FrontImage *string `graphql:"front_image" json:"front_image"` - BackImage *string `graphql:"back_image" json:"back_image"` -} - -type ScrapedMovieStudio struct { - // Set if studio matched - ID *string `graphql:"id" json:"id"` - Name string `graphql:"name" json:"name"` - URL *string `graphql:"url" json:"url"` -} - type ScrapedItems []*ScrapedItem func (s *ScrapedItems) Append(o interface{}) { diff --git a/pkg/scraper/action.go b/pkg/scraper/action.go index ca7e82b2c..fdfa15afa 100644 --- a/pkg/scraper/action.go +++ b/pkg/scraper/action.go @@ -37,10 +37,12 @@ type scraper interface { scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) - scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) + scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) + scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) scrapeSceneByURL(url string) (*models.ScrapedScene, error) - scrapeGalleryByFragment(scene models.GalleryUpdateInput) (*models.ScrapedGallery, error) + scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) + scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) scrapeGalleryByURL(url string) (*models.ScrapedGallery, error) scrapeMovieByURL(url string) (*models.ScrapedMovie, error) diff --git a/pkg/scraper/config.go b/pkg/scraper/config.go index c16d55a7f..d71d2f954 100644 --- a/pkg/scraper/config.go +++ b/pkg/scraper/config.go @@ -393,8 +393,18 @@ func (c config) matchesMovieURL(url string) bool { return false } -func (c config) ScrapeScene(scene models.SceneUpdateInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { +func (c config) ScrapeSceneByScene(scene *models.Scene, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { if c.SceneByFragment != nil { + s := getScraper(*c.SceneByFragment, txnManager, c, globalConfig) + return s.scrapeSceneByScene(scene) + } + + return nil, nil +} + +func (c config) ScrapeSceneByFragment(scene models.ScrapedSceneInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedScene, error) { + if c.SceneByFragment != nil { + // TODO - this should be sceneByQueryFragment s := getScraper(*c.SceneByFragment, txnManager, c, globalConfig) return s.scrapeSceneByFragment(scene) } @@ -420,8 +430,18 @@ func (c config) ScrapeSceneURL(url string, txnManager models.TransactionManager, return nil, nil } -func (c config) ScrapeGallery(gallery models.GalleryUpdateInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { +func (c config) ScrapeGalleryByGallery(gallery *models.Gallery, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { + if c.SceneByFragment != nil { + s := getScraper(*c.GalleryByFragment, txnManager, c, globalConfig) + return s.scrapeGalleryByGallery(gallery) + } + + return nil, nil +} + +func (c config) ScrapeGalleryByFragment(gallery models.ScrapedGalleryInput, txnManager models.TransactionManager, globalConfig GlobalConfig) (*models.ScrapedGallery, error) { if c.GalleryByFragment != nil { + // TODO - this should be galleryByQueryFragment s := getScraper(*c.GalleryByFragment, txnManager, c, globalConfig) return s.scrapeGalleryByFragment(gallery) } diff --git a/pkg/scraper/image.go b/pkg/scraper/image.go index ab09f28da..66261537f 100644 --- a/pkg/scraper/image.go +++ b/pkg/scraper/image.go @@ -28,6 +28,8 @@ func setPerformerImage(p *models.ScrapedPerformer, globalConfig GlobalConfig) er } p.Image = img + // Image is deprecated. Use images instead + p.Images = []string{*img} return nil } diff --git a/pkg/scraper/json.go b/pkg/scraper/json.go index b7c68e86e..e5786761a 100644 --- a/pkg/scraper/json.go +++ b/pkg/scraper/json.go @@ -143,18 +143,9 @@ func (s *jsonScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedP return nil, errors.New("scrapePerformerByFragment not supported for json scraper") } -func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - storedScene, err := sceneFromUpdateFragment(scene, s.txnManager) - if err != nil { - return nil, err - } - - if storedScene == nil { - return nil, errors.New("no scene found") - } - +func (s *jsonScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { // construct the URL - queryURL := queryURLParametersFromScene(storedScene) + queryURL := queryURLParametersFromScene(scene) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -176,18 +167,13 @@ func (s *jsonScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mod return scraper.scrapeScene(q) } -func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager) - if err != nil { - return nil, err - } - - if storedGallery == nil { - return nil, errors.New("no scene found") - } +func (s *jsonScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + return nil, errors.New("scrapeSceneByFragment not supported for json scraper") +} +func (s *jsonScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { // construct the URL - queryURL := queryURLParametersFromGallery(storedGallery) + queryURL := queryURLParametersFromGallery(gallery) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -209,6 +195,10 @@ func (s *jsonScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) return scraper.scrapeGallery(q) } +func (s *jsonScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for json scraper") +} + func (s *jsonScraper) getJsonQuery(doc string) *jsonQuery { return &jsonQuery{ doc: doc, diff --git a/pkg/scraper/mapped.go b/pkg/scraper/mapped.go index 5cbdead74..52af8556c 100644 --- a/pkg/scraper/mapped.go +++ b/pkg/scraper/mapped.go @@ -763,7 +763,7 @@ func (s mappedScraper) scrapePerformer(q mappedQuery) (*models.ScrapedPerformer, tagResults := performerTagsMap.process(q, s.Common) for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -824,11 +824,11 @@ func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) performerResults := scenePerformersMap.process(q, s.Common) for _, p := range performerResults { - performer := &models.ScrapedScenePerformer{} + performer := &models.ScrapedPerformer{} p.apply(performer) for _, p := range performerTagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -842,7 +842,7 @@ func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) tagResults := sceneTagsMap.process(q, s.Common) for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -853,7 +853,7 @@ func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) studioResults := sceneStudioMap.process(q, s.Common) if len(studioResults) > 0 { - studio := &models.ScrapedSceneStudio{} + studio := &models.ScrapedStudio{} studioResults[0].apply(studio) ret.Studio = studio } @@ -864,7 +864,7 @@ func (s mappedScraper) scrapeScene(q mappedQuery) (*models.ScrapedScene, error) movieResults := sceneMoviesMap.process(q, s.Common) for _, p := range movieResults { - movie := &models.ScrapedSceneMovie{} + movie := &models.ScrapedMovie{} p.apply(movie) ret.Movies = append(ret.Movies, movie) } @@ -899,7 +899,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err performerResults := galleryPerformersMap.process(q, s.Common) for _, p := range performerResults { - performer := &models.ScrapedScenePerformer{} + performer := &models.ScrapedPerformer{} p.apply(performer) ret.Performers = append(ret.Performers, performer) } @@ -910,7 +910,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err tagResults := galleryTagsMap.process(q, s.Common) for _, p := range tagResults { - tag := &models.ScrapedSceneTag{} + tag := &models.ScrapedTag{} p.apply(tag) ret.Tags = append(ret.Tags, tag) } @@ -921,7 +921,7 @@ func (s mappedScraper) scrapeGallery(q mappedQuery) (*models.ScrapedGallery, err studioResults := galleryStudioMap.process(q, s.Common) if len(studioResults) > 0 { - studio := &models.ScrapedSceneStudio{} + studio := &models.ScrapedStudio{} studioResults[0].apply(studio) ret.Studio = studio } @@ -951,7 +951,7 @@ func (s mappedScraper) scrapeMovie(q mappedQuery) (*models.ScrapedMovie, error) studioResults := movieStudioMap.process(q, s.Common) if len(studioResults) > 0 { - studio := &models.ScrapedMovieStudio{} + studio := &models.ScrapedStudio{} studioResults[0].apply(studio) ret.Studio = studio } diff --git a/pkg/scraper/matchers.go b/pkg/scraper/matchers.go index cc1d6f99c..fc9bf29e2 100644 --- a/pkg/scraper/matchers.go +++ b/pkg/scraper/matchers.go @@ -7,10 +7,14 @@ import ( "github.com/stashapp/stash/pkg/tag" ) -// MatchScrapedScenePerformer matches the provided performer with the +// MatchScrapedPerformer matches the provided performer with the // performers in the database and sets the ID field if one is found. -func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScenePerformer) error { - performers, err := qb.FindByNames([]string{p.Name}, true) +func MatchScrapedPerformer(qb models.PerformerReader, p *models.ScrapedPerformer) error { + if p.Name == nil { + return nil + } + + performers, err := qb.FindByNames([]string{*p.Name}, true) if err != nil { return err @@ -22,13 +26,13 @@ func MatchScrapedScenePerformer(qb models.PerformerReader, p *models.ScrapedScen } id := strconv.Itoa(performers[0].ID) - p.ID = &id + p.StoredID = &id return nil } -// MatchScrapedSceneStudio matches the provided studio with the studios +// MatchScrapedStudio matches the provided studio with the studios // in the database and sets the ID field if one is found. -func MatchScrapedSceneStudio(qb models.StudioReader, s *models.ScrapedSceneStudio) error { +func MatchScrapedStudio(qb models.StudioReader, s *models.ScrapedStudio) error { studio, err := qb.FindByName(s.Name, true) if err != nil { @@ -41,14 +45,18 @@ func MatchScrapedSceneStudio(qb models.StudioReader, s *models.ScrapedSceneStudi } id := strconv.Itoa(studio.ID) - s.ID = &id + s.StoredID = &id return nil } -// MatchScrapedSceneMovie matches the provided movie with the movies +// MatchScrapedMovie matches the provided movie with the movies // in the database and sets the ID field if one is found. -func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) error { - movies, err := qb.FindByNames([]string{m.Name}, true) +func MatchScrapedMovie(qb models.MovieReader, m *models.ScrapedMovie) error { + if m.Name == nil { + return nil + } + + movies, err := qb.FindByNames([]string{*m.Name}, true) if err != nil { return err @@ -60,13 +68,13 @@ func MatchScrapedSceneMovie(qb models.MovieReader, m *models.ScrapedSceneMovie) } id := strconv.Itoa(movies[0].ID) - m.ID = &id + m.StoredID = &id return nil } -// MatchScrapedSceneTag matches the provided tag with the tags +// MatchScrapedTag matches the provided tag with the tags // in the database and sets the ID field if one is found. -func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error { +func MatchScrapedTag(qb models.TagReader, s *models.ScrapedTag) error { t, err := tag.ByName(qb, s.Name) if err != nil { @@ -87,6 +95,6 @@ func MatchScrapedSceneTag(qb models.TagReader, s *models.ScrapedSceneTag) error } id := strconv.Itoa(t.ID) - s.ID = &id + s.StoredID = &id return nil } diff --git a/pkg/scraper/scrapers.go b/pkg/scraper/scrapers.go index 6c8d6e09d..41b57d2cf 100644 --- a/pkg/scraper/scrapers.go +++ b/pkg/scraper/scrapers.go @@ -3,10 +3,10 @@ package scraper import ( "context" "errors" + "fmt" "os" "path/filepath" "regexp" - "strconv" "strings" "github.com/stashapp/stash/pkg/logger" @@ -260,7 +260,7 @@ func (c Cache) postScrapePerformer(ret *models.ScrapedPerformer) error { return nil } -func (c Cache) postScrapeScenePerformer(ret *models.ScrapedScenePerformer) error { +func (c Cache) postScrapeScenePerformer(ret *models.ScrapedPerformer) error { if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { tqb := r.Tag() @@ -290,13 +290,13 @@ func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { return err } - if err := MatchScrapedScenePerformer(pqb, p); err != nil { + if err := MatchScrapedPerformer(pqb, p); err != nil { return err } } for _, p := range ret.Movies { - err := MatchScrapedSceneMovie(mqb, p) + err := MatchScrapedMovie(mqb, p) if err != nil { return err } @@ -309,7 +309,7 @@ func (c Cache) postScrapeScene(ret *models.ScrapedScene) error { ret.Tags = tags if ret.Studio != nil { - err := MatchScrapedSceneStudio(sqb, ret.Studio) + err := MatchScrapedStudio(sqb, ret.Studio) if err != nil { return err } @@ -335,7 +335,7 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { sqb := r.Studio() for _, p := range ret.Performers { - err := MatchScrapedScenePerformer(pqb, p) + err := MatchScrapedPerformer(pqb, p) if err != nil { return err } @@ -348,7 +348,7 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { ret.Tags = tags if ret.Studio != nil { - err := MatchScrapedSceneStudio(sqb, ret.Studio) + err := MatchScrapedStudio(sqb, ret.Studio) if err != nil { return err } @@ -362,12 +362,42 @@ func (c Cache) postScrapeGallery(ret *models.ScrapedGallery) error { return nil } -// ScrapeScene uses the scraper with the provided ID to scrape a scene. -func (c Cache) ScrapeScene(scraperID string, scene models.SceneUpdateInput) (*models.ScrapedScene, error) { +// ScrapeScene uses the scraper with the provided ID to scrape a scene using existing data. +func (c Cache) ScrapeScene(scraperID string, sceneID int) (*models.ScrapedScene, error) { + // find scraper with the provided id + s := c.findScraper(scraperID) + if s == nil { + return nil, fmt.Errorf("scraper with ID %s not found", scraperID) + } + + // get scene from id + scene, err := getScene(sceneID, c.txnManager) + if err != nil { + return nil, err + } + + ret, err := s.ScrapeSceneByScene(scene, c.txnManager, c.globalConfig) + + if err != nil { + return nil, err + } + + if ret != nil { + err = c.postScrapeScene(ret) + if err != nil { + return nil, err + } + } + + return ret, nil +} + +// ScrapeSceneFragment uses the scraper with the provided ID to scrape a scene. +func (c Cache) ScrapeSceneFragment(scraperID string, scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { // find scraper with the provided id s := c.findScraper(scraperID) if s != nil { - ret, err := s.ScrapeScene(scene, c.txnManager, c.globalConfig) + ret, err := s.ScrapeSceneByFragment(scene, c.txnManager, c.globalConfig) if err != nil { return nil, err @@ -410,11 +440,40 @@ func (c Cache) ScrapeSceneURL(url string) (*models.ScrapedScene, error) { return nil, nil } -// ScrapeGallery uses the scraper with the provided ID to scrape a scene. -func (c Cache) ScrapeGallery(scraperID string, gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { +// ScrapeGallery uses the scraper with the provided ID to scrape a gallery using existing data. +func (c Cache) ScrapeGallery(scraperID string, galleryID int) (*models.ScrapedGallery, error) { s := c.findScraper(scraperID) if s != nil { - ret, err := s.ScrapeGallery(gallery, c.txnManager, c.globalConfig) + // get gallery from id + gallery, err := getGallery(galleryID, c.txnManager) + if err != nil { + return nil, err + } + + ret, err := s.ScrapeGalleryByGallery(gallery, c.txnManager, c.globalConfig) + + if err != nil { + return nil, err + } + + if ret != nil { + err = c.postScrapeGallery(ret) + if err != nil { + return nil, err + } + } + + return ret, nil + } + + return nil, errors.New("Scraped with ID " + scraperID + " not found") +} + +// ScrapeGalleryFragment uses the scraper with the provided ID to scrape a gallery. +func (c Cache) ScrapeGalleryFragment(scraperID string, gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + s := c.findScraper(scraperID) + if s != nil { + ret, err := s.ScrapeGalleryByFragment(gallery, c.txnManager, c.globalConfig) if err != nil { return nil, err @@ -457,23 +516,6 @@ func (c Cache) ScrapeGalleryURL(url string) (*models.ScrapedGallery, error) { return nil, nil } -func matchMovieStudio(qb models.StudioReader, s *models.ScrapedMovieStudio) error { - studio, err := qb.FindByName(s.Name, true) - - if err != nil { - return err - } - - if studio == nil { - // ignore - cannot match - return nil - } - - id := strconv.Itoa(studio.ID) - s.ID = &id - return nil -} - // ScrapeMovieURL uses the first scraper it finds that matches the URL // provided to scrape a movie. If no scrapers are found that matches // the URL, then nil is returned. @@ -487,7 +529,7 @@ func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error) { if ret.Studio != nil { if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - return matchMovieStudio(r.Studio(), ret.Studio) + return MatchScrapedStudio(r.Studio(), ret.Studio) }); err != nil { return nil, err } @@ -508,8 +550,8 @@ func (c Cache) ScrapeMovieURL(url string) (*models.ScrapedMovie, error) { return nil, nil } -func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedSceneTag) ([]*models.ScrapedSceneTag, error) { - var ret []*models.ScrapedSceneTag +func postProcessTags(tqb models.TagReader, scrapedTags []*models.ScrapedTag) ([]*models.ScrapedTag, error) { + var ret []*models.ScrapedTag excludePatterns := stash_config.GetInstance().GetScraperExcludeTagPatterns() var excludeRegexps []*regexp.Regexp @@ -533,7 +575,7 @@ ScrapeTag: } } - err := MatchScrapedSceneTag(tqb, t) + err := MatchScrapedTag(tqb, t) if err != nil { return nil, err } diff --git a/pkg/scraper/script.go b/pkg/scraper/script.go index 32f768d45..6b47dd6ef 100644 --- a/pkg/scraper/script.go +++ b/pkg/scraper/script.go @@ -63,7 +63,7 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error if err = cmd.Start(); err != nil { logger.Error("Error running scraper script: " + err.Error()) - return errors.New("Error running scraper script") + return errors.New("error running scraper script") } scanner := bufio.NewScanner(stderr) @@ -86,7 +86,7 @@ func (s *scriptScraper) runScraperScript(inString string, out interface{}) error logger.Debugf("Scraper script finished") if err != nil { - return errors.New("Error running scraper script") + return errors.New("error running scraper script") } return nil @@ -134,7 +134,21 @@ func (s *scriptScraper) scrapePerformerByURL(url string) (*models.ScrapedPerform return &ret, err } -func (s *scriptScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { +func (s *scriptScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { + inString, err := json.Marshal(sceneToUpdateInput(scene)) + + if err != nil { + return nil, err + } + + var ret models.ScrapedScene + + err = s.runScraperScript(string(inString), &ret) + + return &ret, err +} + +func (s *scriptScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { inString, err := json.Marshal(scene) if err != nil { @@ -148,7 +162,21 @@ func (s *scriptScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*m return &ret, err } -func (s *scriptScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { +func (s *scriptScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { + inString, err := json.Marshal(galleryToUpdateInput(gallery)) + + if err != nil { + return nil, err + } + + var ret models.ScrapedGallery + + err = s.runScraperScript(string(inString), &ret) + + return &ret, err +} + +func (s *scriptScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { inString, err := json.Marshal(gallery) if err != nil { diff --git a/pkg/scraper/stash.go b/pkg/scraper/stash.go index d37b82847..539057196 100644 --- a/pkg/scraper/stash.go +++ b/pkg/scraper/stash.go @@ -2,6 +2,7 @@ package scraper import ( "context" + "database/sql" "errors" "strconv" @@ -81,11 +82,40 @@ func (s *stashScraper) scrapePerformersByName(name string) ([]*models.ScrapedPer return ret, nil } +// need a separate for scraped stash performers - does not include remote_site_id or image +type scrapedTagStash struct { + Name string `graphql:"name" json:"name"` +} + +type scrapedPerformerStash struct { + Name *string `graphql:"name" json:"name"` + Gender *string `graphql:"gender" json:"gender"` + URL *string `graphql:"url" json:"url"` + Twitter *string `graphql:"twitter" json:"twitter"` + Instagram *string `graphql:"instagram" json:"instagram"` + Birthdate *string `graphql:"birthdate" json:"birthdate"` + Ethnicity *string `graphql:"ethnicity" json:"ethnicity"` + Country *string `graphql:"country" json:"country"` + EyeColor *string `graphql:"eye_color" json:"eye_color"` + Height *string `graphql:"height" json:"height"` + Measurements *string `graphql:"measurements" json:"measurements"` + FakeTits *string `graphql:"fake_tits" json:"fake_tits"` + CareerLength *string `graphql:"career_length" json:"career_length"` + Tattoos *string `graphql:"tattoos" json:"tattoos"` + Piercings *string `graphql:"piercings" json:"piercings"` + Aliases *string `graphql:"aliases" json:"aliases"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Details *string `graphql:"details" json:"details"` + DeathDate *string `graphql:"death_date" json:"death_date"` + HairColor *string `graphql:"hair_color" json:"hair_color"` + Weight *string `graphql:"weight" json:"weight"` +} + func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) { client := s.getStashClient() var q struct { - FindPerformer *models.ScrapedPerformerStash `graphql:"findPerformer(id: $f)"` + FindPerformer *scrapedPerformerStash `graphql:"findPerformer(id: $f)"` } performerID := *scrapedPerformer.URL @@ -100,13 +130,6 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return nil, err } - if q.FindPerformer != nil { - // the ids of the tags must be nilled - for _, t := range q.FindPerformer.Tags { - t.ID = nil - } - } - // need to copy back to a scraped performer ret := models.ScrapedPerformer{} err = copier.Copy(&ret, q.FindPerformer) @@ -123,25 +146,27 @@ func (s *stashScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return &ret, nil } -func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { +type scrapedStudioStash struct { + Name string `graphql:"name" json:"name"` + URL *string `graphql:"url" json:"url"` +} + +type scrapedSceneStash struct { + ID string `graphql:"id" json:"id"` + Title *string `graphql:"title" json:"title"` + Details *string `graphql:"details" json:"details"` + URL *string `graphql:"url" json:"url"` + Date *string `graphql:"date" json:"date"` + File *models.SceneFileType `graphql:"file" json:"file"` + Studio *scrapedStudioStash `graphql:"studio" json:"studio"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"` +} + +func (s *stashScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { // query by MD5 - // assumes that the scene exists in the database - id, err := strconv.Atoi(scene.ID) - if err != nil { - return nil, err - } - - var storedScene *models.Scene - if err := s.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - var err error - storedScene, err = r.Scene().Find(id) - return err - }); err != nil { - return nil, err - } - var q struct { - FindScene *models.ScrapedSceneStash `graphql:"findSceneByHash(input: $c)"` + FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"` } type SceneHashInput struct { @@ -150,8 +175,8 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo } input := SceneHashInput{ - Checksum: &storedScene.Checksum.String, - Oshash: &storedScene.OSHash.String, + Checksum: &scene.Checksum.String, + Oshash: &scene.OSHash.String, } vars := map[string]interface{}{ @@ -159,34 +184,18 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo } client := s.getStashClient() - err = client.Query(context.Background(), &q, vars) - if err != nil { + if err := client.Query(context.Background(), &q, vars); err != nil { return nil, err } - if q.FindScene != nil { - // the ids of the studio, performers and tags must be nilled - if q.FindScene.Studio != nil { - q.FindScene.Studio.ID = nil - } - - for _, p := range q.FindScene.Performers { - p.ID = nil - } - - for _, t := range q.FindScene.Tags { - t.ID = nil - } - } - // need to copy back to a scraped scene ret := models.ScrapedScene{} - err = copier.Copy(&ret, q.FindScene) - if err != nil { + if err := copier.Copy(&ret, q.FindScene); err != nil { return nil, err } // get the performer image directly + var err error ret.Image, err = getStashSceneImage(s.config.StashServer.URL, q.FindScene.ID, s.globalConfig) if err != nil { return nil, err @@ -195,27 +204,25 @@ func (s *stashScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo return &ret, nil } -func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - id, err := strconv.Atoi(scene.ID) - if err != nil { - return nil, err - } +func (s *stashScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + return nil, errors.New("scrapeSceneByFragment not supported for stash scraper") +} - // query by MD5 - // assumes that the gallery exists in the database - var storedGallery *models.Gallery - if err := s.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - qb := r.Gallery() - - var err error - storedGallery, err = qb.Find(id) - return err - }); err != nil { - return nil, err - } +type scrapedGalleryStash struct { + ID string `graphql:"id" json:"id"` + Title *string `graphql:"title" json:"title"` + Details *string `graphql:"details" json:"details"` + URL *string `graphql:"url" json:"url"` + Date *string `graphql:"date" json:"date"` + File *models.SceneFileType `graphql:"file" json:"file"` + Studio *scrapedStudioStash `graphql:"studio" json:"studio"` + Tags []*scrapedTagStash `graphql:"tags" json:"tags"` + Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"` +} +func (s *stashScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { var q struct { - FindGallery *models.ScrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"` + FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"` } type GalleryHashInput struct { @@ -223,7 +230,7 @@ func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) } input := GalleryHashInput{ - Checksum: &storedGallery.Checksum, + Checksum: &gallery.Checksum, } vars := map[string]interface{}{ @@ -231,36 +238,23 @@ func (s *stashScraper) scrapeGalleryByFragment(scene models.GalleryUpdateInput) } client := s.getStashClient() - err = client.Query(context.Background(), &q, vars) - if err != nil { + if err := client.Query(context.Background(), &q, vars); err != nil { return nil, err } - if q.FindGallery != nil { - // the ids of the studio, performers and tags must be nilled - if q.FindGallery.Studio != nil { - q.FindGallery.Studio.ID = nil - } - - for _, p := range q.FindGallery.Performers { - p.ID = nil - } - - for _, t := range q.FindGallery.Tags { - t.ID = nil - } - } - // need to copy back to a scraped scene ret := models.ScrapedGallery{} - err = copier.Copy(&ret, q.FindGallery) - if err != nil { + if err := copier.Copy(&ret, q.FindGallery); err != nil { return nil, err } return &ret, nil } +func (s *stashScraper) scrapeGalleryByFragment(scene models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for stash scraper") +} + func (s *stashScraper) scrapePerformerByURL(url string) (*models.ScrapedPerformer, error) { return nil, errors.New("scrapePerformerByURL not supported for stash scraper") } @@ -277,17 +271,11 @@ func (s *stashScraper) scrapeMovieByURL(url string) (*models.ScrapedMovie, error return nil, errors.New("scrapeMovieByURL not supported for stash scraper") } -func sceneFromUpdateFragment(scene models.SceneUpdateInput, txnManager models.TransactionManager) (*models.Scene, error) { - id, err := strconv.Atoi(scene.ID) - if err != nil { - return nil, err - } - - // TODO - should we modify it with the input? +func getScene(sceneID int, txnManager models.TransactionManager) (*models.Scene, error) { var ret *models.Scene if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { var err error - ret, err = r.Scene().Find(id) + ret, err = r.Scene().Find(sceneID) return err }); err != nil { return nil, err @@ -295,18 +283,66 @@ func sceneFromUpdateFragment(scene models.SceneUpdateInput, txnManager models.Tr return ret, nil } -func galleryFromUpdateFragment(gallery models.GalleryUpdateInput, txnManager models.TransactionManager) (ret *models.Gallery, err error) { - id, err := strconv.Atoi(gallery.ID) - if err != nil { - return nil, err +func sceneToUpdateInput(scene *models.Scene) models.SceneUpdateInput { + toStringPtr := func(s sql.NullString) *string { + if s.Valid { + return &s.String + } + + return nil } + dateToStringPtr := func(s models.SQLiteDate) *string { + if s.Valid { + return &s.String + } + + return nil + } + + return models.SceneUpdateInput{ + ID: strconv.Itoa(scene.ID), + Title: toStringPtr(scene.Title), + Details: toStringPtr(scene.Details), + URL: toStringPtr(scene.URL), + Date: dateToStringPtr(scene.Date), + } +} + +func getGallery(galleryID int, txnManager models.TransactionManager) (*models.Gallery, error) { + var ret *models.Gallery if err := txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { - ret, err = r.Gallery().Find(id) + var err error + ret, err = r.Gallery().Find(galleryID) return err }); err != nil { return nil, err } - return ret, nil } + +func galleryToUpdateInput(gallery *models.Gallery) models.GalleryUpdateInput { + toStringPtr := func(s sql.NullString) *string { + if s.Valid { + return &s.String + } + + return nil + } + + dateToStringPtr := func(s models.SQLiteDate) *string { + if s.Valid { + return &s.String + } + + return nil + } + + return models.GalleryUpdateInput{ + ID: strconv.Itoa(gallery.ID), + Title: toStringPtr(gallery.Title), + Details: toStringPtr(gallery.Details), + URL: toStringPtr(gallery.URL), + Date: dateToStringPtr(gallery.Date), + } +} diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 4ab848fbf..f7bd53a99 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -66,8 +66,79 @@ func (c Client) QueryStashBoxScene(queryStr string) ([]*models.ScrapedScene, err } // FindStashBoxScenesByFingerprints queries stash-box for scenes using every -// scene's MD5/OSHASH checksum, or PHash -func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.ScrapedScene, error) { +// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order +// as the input slice. +func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models.ScrapedScene, error) { + ids, err := utils.StringSliceToIntSlice(sceneIDs) + if err != nil { + return nil, err + } + + var fingerprints []string + // map fingerprints to their scene index + fpToScene := make(map[string][]int) + + if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + qb := r.Scene() + + for index, sceneID := range ids { + scene, err := qb.Find(sceneID) + if err != nil { + return err + } + + if scene == nil { + return fmt.Errorf("scene with id %d not found", sceneID) + } + + if scene.Checksum.Valid { + fingerprints = append(fingerprints, scene.Checksum.String) + fpToScene[scene.Checksum.String] = append(fpToScene[scene.Checksum.String], index) + } + + if scene.OSHash.Valid { + fingerprints = append(fingerprints, scene.OSHash.String) + fpToScene[scene.OSHash.String] = append(fpToScene[scene.OSHash.String], index) + } + + if scene.Phash.Valid { + phashStr := utils.PhashToString(scene.Phash.Int64) + fingerprints = append(fingerprints, phashStr) + fpToScene[phashStr] = append(fpToScene[phashStr], index) + } + } + + return nil + }); err != nil { + return nil, err + } + + allScenes, err := c.findStashBoxScenesByFingerprints(fingerprints) + if err != nil { + return nil, err + } + + // set the matched scenes back in their original order + ret := make([][]*models.ScrapedScene, len(sceneIDs)) + for _, s := range allScenes { + var addedTo []int + for _, fp := range s.Fingerprints { + sceneIndexes := fpToScene[fp.Hash] + for _, index := range sceneIndexes { + if !utils.IntInclude(addedTo, index) { + addedTo = append(addedTo, index) + ret[index] = append(ret[index], s) + } + } + } + } + + return ret, nil +} + +// FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every +// scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice. +func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*models.ScrapedScene, error) { ids, err := utils.StringSliceToIntSlice(sceneIDs) if err != nil { return nil, err @@ -97,7 +168,8 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.S } if scene.Phash.Valid { - fingerprints = append(fingerprints, utils.PhashToString(scene.Phash.Int64)) + phashStr := utils.PhashToString(scene.Phash.Int64) + fingerprints = append(fingerprints, phashStr) } } @@ -237,10 +309,18 @@ func (c Client) QueryStashBoxPerformer(queryStr string) ([]*models.StashBoxPerfo Results: performers, }, } + + // set the deprecated image field + for _, p := range res[0].Results { + if len(p.Images) > 0 { + p.Image = &p.Images[0] + } + } + return res, err } -func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedScenePerformer, error) { +func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedPerformer, error) { performers, err := c.client.SearchPerformer(context.TODO(), queryStr) if err != nil { return nil, err @@ -248,7 +328,7 @@ func (c Client) queryStashBoxPerformer(queryStr string) ([]*models.ScrapedSceneP performerFragments := performers.SearchPerformer - var ret []*models.ScrapedScenePerformer + var ret []*models.ScrapedPerformer for _, fragment := range performerFragments { performer := performerFragmentToScrapedScenePerformer(*fragment) ret = append(ret, performer) @@ -292,6 +372,50 @@ func (c Client) FindStashBoxPerformersByNames(performerIDs []string) ([]*models. return c.findStashBoxPerformersByNames(performers) } +func (c Client) FindStashBoxPerformersByPerformerNames(performerIDs []string) ([][]*models.ScrapedPerformer, error) { + ids, err := utils.StringSliceToIntSlice(performerIDs) + if err != nil { + return nil, err + } + + var performers []*models.Performer + + if err := c.txnManager.WithReadTxn(context.TODO(), func(r models.ReaderRepository) error { + qb := r.Performer() + + for _, performerID := range ids { + performer, err := qb.Find(performerID) + if err != nil { + return err + } + + if performer == nil { + return fmt.Errorf("performer with id %d not found", performerID) + } + + if performer.Name.Valid { + performers = append(performers, performer) + } + } + + return nil + }); err != nil { + return nil, err + } + + results, err := c.findStashBoxPerformersByNames(performers) + if err != nil { + return nil, err + } + + var ret [][]*models.ScrapedPerformer + for _, r := range results { + ret = append(ret, r.Results) + } + + return ret, nil +} + func (c Client) findStashBoxPerformersByNames(performers []*models.Performer) ([]*models.StashBoxPerformerQueryResult, error) { var ret []*models.StashBoxPerformerQueryResult for _, performer := range performers { @@ -413,14 +537,14 @@ func fetchImage(url string) (*string, error) { return &img, nil } -func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *models.ScrapedScenePerformer { +func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *models.ScrapedPerformer { id := p.ID images := []string{} for _, image := range p.Images { images = append(images, image.URL) } - sp := &models.ScrapedScenePerformer{ - Name: p.Name, + sp := &models.ScrapedPerformer{ + Name: &p.Name, Country: p.Country, Measurements: formatMeasurements(p.Measurements), CareerLength: formatCareerLength(p.CareerStartYear, p.CareerEndYear), @@ -430,10 +554,13 @@ func performerFragmentToScrapedScenePerformer(p graphql.PerformerFragment) *mode RemoteSiteID: &id, Images: images, // TODO - tags not currently supported - // TODO - Image - should be returned as a set of URLs. Will need a // graphql schema change to accommodate this. Leave off for now. } + if len(sp.Images) > 0 { + sp.Image = &sp.Images[0] + } + if p.Height != nil && *p.Height > 0 { hs := strconv.Itoa(*p.Height) sp.Height = &hs @@ -511,13 +638,13 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq if s.Studio != nil { studioID := s.Studio.ID - ss.Studio = &models.ScrapedSceneStudio{ + ss.Studio = &models.ScrapedStudio{ Name: s.Studio.Name, URL: findURL(s.Studio.Urls, "HOME"), RemoteSiteID: &studioID, } - err := scraper.MatchScrapedSceneStudio(r.Studio(), ss.Studio) + err := scraper.MatchScrapedStudio(r.Studio(), ss.Studio) if err != nil { return err } @@ -526,7 +653,7 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq for _, p := range s.Performers { sp := performerFragmentToScrapedScenePerformer(p.Performer) - err := scraper.MatchScrapedScenePerformer(pqb, sp) + err := scraper.MatchScrapedPerformer(pqb, sp) if err != nil { return err } @@ -535,11 +662,11 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq } for _, t := range s.Tags { - st := &models.ScrapedSceneTag{ + st := &models.ScrapedTag{ Name: t.Name, } - err := scraper.MatchScrapedSceneTag(tqb, st) + err := scraper.MatchScrapedTag(tqb, st) if err != nil { return err } @@ -555,7 +682,7 @@ func sceneFragmentToScrapedScene(txnManager models.TransactionManager, s *graphq return ss, nil } -func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedScenePerformer, error) { +func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedPerformer, error) { performer, err := c.client.FindPerformerByID(context.TODO(), id) if err != nil { return nil, err @@ -565,13 +692,13 @@ func (c Client) FindStashBoxPerformerByID(id string) (*models.ScrapedScenePerfor return ret, nil } -func (c Client) FindStashBoxPerformerByName(name string) (*models.ScrapedScenePerformer, error) { +func (c Client) FindStashBoxPerformerByName(name string) (*models.ScrapedPerformer, error) { performers, err := c.client.SearchPerformer(context.TODO(), name) if err != nil { return nil, err } - var ret *models.ScrapedScenePerformer + var ret *models.ScrapedPerformer for _, performer := range performers.SearchPerformer { if strings.EqualFold(performer.Name, name) { ret = performerFragmentToScrapedScenePerformer(*performer) diff --git a/pkg/scraper/xpath.go b/pkg/scraper/xpath.go index e612b5f4d..71ab74a8d 100644 --- a/pkg/scraper/xpath.go +++ b/pkg/scraper/xpath.go @@ -124,18 +124,9 @@ func (s *xpathScraper) scrapePerformerByFragment(scrapedPerformer models.Scraped return nil, errors.New("scrapePerformerByFragment not supported for xpath scraper") } -func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*models.ScrapedScene, error) { - storedScene, err := sceneFromUpdateFragment(scene, s.txnManager) - if err != nil { - return nil, err - } - - if storedScene == nil { - return nil, errors.New("no scene found") - } - +func (s *xpathScraper) scrapeSceneByScene(scene *models.Scene) (*models.ScrapedScene, error) { // construct the URL - queryURL := queryURLParametersFromScene(storedScene) + queryURL := queryURLParametersFromScene(scene) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -157,18 +148,13 @@ func (s *xpathScraper) scrapeSceneByFragment(scene models.SceneUpdateInput) (*mo return scraper.scrapeScene(q) } -func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput) (*models.ScrapedGallery, error) { - storedGallery, err := galleryFromUpdateFragment(gallery, s.txnManager) - if err != nil { - return nil, err - } - - if storedGallery == nil { - return nil, errors.New("no scene found") - } +func (s *xpathScraper) scrapeSceneByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) { + return nil, errors.New("scrapeSceneByFragment not supported for xpath scraper") +} +func (s *xpathScraper) scrapeGalleryByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) { // construct the URL - queryURL := queryURLParametersFromGallery(storedGallery) + queryURL := queryURLParametersFromGallery(gallery) if s.scraper.QueryURLReplacements != nil { queryURL.applyReplacements(s.scraper.QueryURLReplacements) } @@ -190,6 +176,10 @@ func (s *xpathScraper) scrapeGalleryByFragment(gallery models.GalleryUpdateInput return scraper.scrapeGallery(q) } +func (s *xpathScraper) scrapeGalleryByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) { + return nil, errors.New("scrapeGalleryByFragment not supported for xpath scraper") +} + func (s *xpathScraper) loadURL(url string) (*html.Node, error) { r, err := loadURL(url, s.config, s.globalConfig) if err != nil { diff --git a/pkg/scraper/xpath_test.go b/pkg/scraper/xpath_test.go index 5983bd7a0..60fb5749f 100644 --- a/pkg/scraper/xpath_test.go +++ b/pkg/scraper/xpath_test.go @@ -593,7 +593,7 @@ func makeSceneXPathConfig() mappedScraper { return scraper } -func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.ScrapedSceneTag) { +func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.ScrapedTag) { t.Helper() i := 0 @@ -614,7 +614,7 @@ func verifyTags(t *testing.T, expectedTagNames []string, actualTags []*models.Sc } } -func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*models.ScrapedSceneMovie) { +func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*models.ScrapedMovie) { t.Helper() i := 0 @@ -625,7 +625,7 @@ func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*mod expectedMovie = expectedMovieNames[i] } if i < len(actualMovies) { - actualMovie = actualMovies[i].Name + actualMovie = *actualMovies[i].Name } if expectedMovie != actualMovie { @@ -635,7 +635,7 @@ func verifyMovies(t *testing.T, expectedMovieNames []string, actualMovies []*mod } } -func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []string, actualPerformers []*models.ScrapedScenePerformer) { +func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []string, actualPerformers []*models.ScrapedPerformer) { t.Helper() i := 0 @@ -651,7 +651,7 @@ func verifyPerformers(t *testing.T, expectedNames []string, expectedURLs []strin expectedURL = expectedURLs[i] } if i < len(actualPerformers) { - actualName = actualPerformers[i].Name + actualName = *actualPerformers[i].Name if actualPerformers[i].URL != nil { actualURL = *actualPerformers[i].URL } diff --git a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx index c83b94bc7..cd13c59df 100644 --- a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx +++ b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryEditPanel.tsx @@ -229,19 +229,18 @@ export const GalleryEditPanel: React.FC< } async function onScrapeClicked(scraper: GQL.Scraper) { + if (!gallery) return; + setIsLoading(true); try { - const galleryInput = getGalleryInput( - formik.values - ) as GQL.GalleryUpdateInput; - const result = await queryScrapeGallery(scraper.id, galleryInput); - if (!result.data || !result.data.scrapeGallery) { + const result = await queryScrapeGallery(scraper.id, gallery.id); + if (!result.data || !result.data.scrapeSingleGallery?.length) { Toast.success({ content: "No galleries found", }); return; } - setScrapedGallery(result.data.scrapeGallery); + setScrapedGallery(result.data.scrapeSingleGallery[0]); } catch (e) { Toast.error(e); } finally { diff --git a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryScrapeDialog.tsx b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryScrapeDialog.tsx index 701bbe019..2e34fc7af 100644 --- a/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryScrapeDialog.tsx +++ b/ui/v2.5/src/components/Galleries/GalleryDetails/GalleryScrapeDialog.tsx @@ -45,8 +45,8 @@ function renderScrapedStudioRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newStudio?: GQL.ScrapedSceneStudio, - onCreateNew?: (value: GQL.ScrapedSceneStudio) => void + newStudio?: GQL.ScrapedStudio, + onCreateNew?: (value: GQL.ScrapedStudio) => void ) { return ( , onChange: (value: ScrapeResult) => void, - newPerformers: GQL.ScrapedScenePerformer[], - onCreateNew?: (value: GQL.ScrapedScenePerformer) => void + newPerformers: GQL.ScrapedPerformer[], + onCreateNew?: (value: GQL.ScrapedPerformer) => void ) { + const performersCopy = newPerformers.map((p) => { + const name: string = p.name ?? ""; + return { ...p, name }; + }); + return ( ); @@ -139,8 +144,8 @@ function renderScrapedTagsRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newTags: GQL.ScrapedSceneTag[], - onCreateNew?: (value: GQL.ScrapedSceneTag) => void + newTags: GQL.ScrapedTag[], + onCreateNew?: (value: GQL.ScrapedTag) => void ) { return ( = ( props.scraped.studio?.stored_id ) ); - const [newStudio, setNewStudio] = useState< - GQL.ScrapedSceneStudio | undefined - >( + const [newStudio, setNewStudio] = useState( props.scraped.studio && !props.scraped.studio.stored_id ? props.scraped.studio : undefined @@ -241,9 +244,9 @@ export const GalleryScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.performers ?? undefined) ) ); - const [newPerformers, setNewPerformers] = useState< - GQL.ScrapedScenePerformer[] - >(props.scraped.performers?.filter((t) => !t.stored_id) ?? []); + const [newPerformers, setNewPerformers] = useState( + props.scraped.performers?.filter((t) => !t.stored_id) ?? [] + ); const [tags, setTags] = useState>( new ScrapeResult( @@ -251,7 +254,7 @@ export const GalleryScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.tags ?? undefined) ) ); - const [newTags, setNewTags] = useState( + const [newTags, setNewTags] = useState( props.scraped.tags?.filter((t) => !t.stored_id) ?? [] ); @@ -275,7 +278,7 @@ export const GalleryScrapeDialog: React.FC = ( return <>; } - async function createNewStudio(toCreate: GQL.ScrapedSceneStudio) { + async function createNewStudio(toCreate: GQL.ScrapedStudio) { try { const result = await createStudio({ variables: { @@ -308,7 +311,7 @@ export const GalleryScrapeDialog: React.FC = ( } } - async function createNewPerformer(toCreate: GQL.ScrapedScenePerformer) { + async function createNewPerformer(toCreate: GQL.ScrapedPerformer) { const input = makePerformerCreateInput(toCreate); try { @@ -349,7 +352,7 @@ export const GalleryScrapeDialog: React.FC = ( } } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ diff --git a/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx b/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx index 24d7f3af1..981d52751 100644 --- a/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx +++ b/ui/v2.5/src/components/Movies/MovieDetails/MovieEditPanel.tsx @@ -203,8 +203,8 @@ export const MovieEditPanel: React.FC = ({ formik.setFieldValue("date", state.date ?? undefined); } - if (state.studio && state.studio.id) { - formik.setFieldValue("studio_id", state.studio.id ?? undefined); + if (state.studio && state.studio.stored_id) { + formik.setFieldValue("studio_id", state.studio.stored_id ?? undefined); } if (state.director) { diff --git a/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx b/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx index aad6735b6..467fa96a7 100644 --- a/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx +++ b/ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx @@ -87,7 +87,10 @@ export const MovieScrapeDialog: React.FC = ( new ScrapeResult(props.movie.synopsis, props.scraped.synopsis) ); const [studio, setStudio] = useState>( - new ScrapeResult(props.movie.studio_id, props.scraped.studio?.id) + new ScrapeResult( + props.movie.studio_id, + props.scraped.studio?.stored_id + ) ); const [url, setURL] = useState>( new ScrapeResult(props.movie.url, props.scraped.url) @@ -123,7 +126,7 @@ export const MovieScrapeDialog: React.FC = ( const durationString = duration.getNewValue(); return { - name: name.getNewValue(), + name: name.getNewValue() ?? "", aliases: aliases.getNewValue(), duration: durationString, date: date.getNewValue(), @@ -131,7 +134,7 @@ export const MovieScrapeDialog: React.FC = ( synopsis: synopsis.getNewValue(), studio: newStudio ? { - id: newStudio, + stored_id: newStudio, name: "", } : undefined, diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx index d62743307..948ad7c86 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerEditPanel.tsx @@ -72,7 +72,7 @@ export const PerformerEditPanel: React.FC = ({ // Editing state const [scraper, setScraper] = useState(); - const [newTags, setNewTags] = useState(); + const [newTags, setNewTags] = useState(); const [isScraperModalOpen, setIsScraperModalOpen] = useState(false); const [isDeleteAlertOpen, setIsDeleteAlertOpen] = useState(false); @@ -224,7 +224,7 @@ export const PerformerEditPanel: React.FC = ({ return ret; } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ @@ -334,9 +334,10 @@ export const PerformerEditPanel: React.FC = ({ // otherwise follow existing behaviour (`undefined`) if ( (!isNew || [null, undefined].includes(formik.values.image)) && - state.image !== undefined + state.images && + state.images.length > 0 ) { - const imageStr = state.image; + const imageStr = state.images[0]; formik.setFieldValue("image", imageStr ?? undefined); } if (state.details) { @@ -524,20 +525,23 @@ export const PerformerEditPanel: React.FC = ({ const { __typename, - image: _image, + images: _image, tags: _tags, ...ret } = selectedPerformer; const result = await queryScrapePerformer(selectedScraper.id, ret); - if (!result?.data?.scrapePerformer) return; + if (!result?.data?.scrapeSinglePerformer?.length) return; + // assume one result // if this is a new performer, just dump the data if (isNew) { - updatePerformerEditStateFromScraper(result.data.scrapePerformer); + updatePerformerEditStateFromScraper( + result.data.scrapeSinglePerformer[0] + ); setScraper(undefined); } else { - setScrapedPerformer(result.data.scrapePerformer); + setScrapedPerformer(result.data.scrapeSinglePerformer[0]); } } catch (e) { Toast.error(e); @@ -569,12 +573,12 @@ export const PerformerEditPanel: React.FC = ({ } } - async function onScrapeStashBox(performerResult: GQL.ScrapedScenePerformer) { + async function onScrapeStashBox(performerResult: GQL.ScrapedPerformer) { setIsScraperModalOpen(false); - const result: Partial = { + const result: GQL.ScrapedPerformerDataFragment = { ...performerResult, - image: performerResult.images?.[0] ?? undefined, + images: performerResult.images ?? undefined, country: getCountryByISO(performerResult.country), __typename: "ScrapedPerformer", }; diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx index 2a48fca8c..d337fc5c7 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx @@ -97,8 +97,8 @@ function renderScrapedTagsRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newTags: GQL.ScrapedSceneTag[], - onCreateNew?: (value: GQL.ScrapedSceneTag) => void + newTags: GQL.ScrapedTag[], + onCreateNew?: (value: GQL.ScrapedTag) => void ) { return ( = ( ) ); - const [newTags, setNewTags] = useState( + const [newTags, setNewTags] = useState( props.scraped.tags?.filter((t) => !t.stored_id) ?? [] ); const [image, setImage] = useState>( - new ScrapeResult(props.performer.image, props.scraped.image) + new ScrapeResult( + props.performer.image, + props.scraped.images && props.scraped.images.length > 0 + ? props.scraped.images[0] + : undefined + ) ); const allFields = [ @@ -338,7 +343,7 @@ export const PerformerScrapeDialog: React.FC = ( return <>; } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ @@ -375,8 +380,9 @@ export const PerformerScrapeDialog: React.FC = ( } function makeNewScrapedItem(): GQL.ScrapedPerformer { + const newImage = image.getNewValue(); return { - name: name.getNewValue(), + name: name.getNewValue() ?? "", aliases: aliases.getNewValue(), birthdate: birthdate.getNewValue(), ethnicity: ethnicity.getNewValue(), @@ -398,7 +404,7 @@ export const PerformerScrapeDialog: React.FC = ( name: "", }; }), - image: image.getNewValue(), + images: newImage ? [newImage] : undefined, details: details.getNewValue(), death_date: deathDate.getNewValue(), hair_color: hairColor.getNewValue(), diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx index 823fa4d25..5c02f1f42 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeModal.tsx @@ -30,7 +30,7 @@ const PerformerScrapeModal: React.FC = ({ const [query, setQuery] = useState(name ?? ""); const { data, loading } = useScrapePerformerList(scraper.id, query); - const performers = data?.scrapePerformerList ?? []; + const performers = data?.scrapeSinglePerformer ?? []; const onInputChange = debounce((input: string) => { setQuery(input); diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx index 319bb73f6..734c2be18 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/PerformerStashBoxModal.tsx @@ -16,7 +16,7 @@ export interface IStashBox extends GQL.StashBox { interface IProps { instance: IStashBox; onHide: () => void; - onSelectPerformer: (performer: GQL.ScrapedScenePerformer) => void; + onSelectPerformer: (performer: GQL.ScrapedPerformer) => void; name?: string; } const PerformerStashBoxModal: React.FC = ({ @@ -28,17 +28,19 @@ const PerformerStashBoxModal: React.FC = ({ const intl = useIntl(); const inputRef = useRef(null); const [query, setQuery] = useState(name ?? ""); - const { data, loading } = GQL.useQueryStashBoxPerformerQuery({ + const { data, loading } = GQL.useScrapeSinglePerformerQuery({ variables: { - input: { + source: { stash_box_index: instance.index, - q: query, + }, + input: { + query, }, }, skip: query === "", }); - const performers = data?.queryStashBoxPerformer?.[0].results ?? []; + const performers = data?.scrapeSinglePerformer ?? []; const onInputChange = debounce((input: string) => { setQuery(input); diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx index f38749729..c08d5060e 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneEditPanel.tsx @@ -277,12 +277,12 @@ export const SceneEditPanel: React.FC = ({ setIsLoading(true); try { const result = await queryStashBoxScene(stashBoxIndex, scene.id); - if (!result.data || !result.data.queryStashBoxScene) { + if (!result.data || !result.data.scrapeSingleScene) { return; } - if (result.data.queryStashBoxScene.length > 0) { - setScrapedScene(result.data.queryStashBoxScene[0]); + if (result.data.scrapeSingleScene.length > 0) { + setScrapedScene(result.data.scrapeSingleScene[0]); } else { Toast.success({ content: "No scenes found", @@ -298,17 +298,15 @@ export const SceneEditPanel: React.FC = ({ async function onScrapeClicked(scraper: GQL.Scraper) { setIsLoading(true); try { - const result = await queryScrapeScene( - scraper.id, - getSceneInput(formik.values) - ); - if (!result.data || !result.data.scrapeScene) { + const result = await queryScrapeScene(scraper.id, scene.id); + if (!result.data || !result.data.scrapeSingleScene?.length) { Toast.success({ content: "No scenes found", }); return; } - setScrapedScene(result.data.scrapeScene); + // assume one returned scene + setScrapedScene(result.data.scrapeSingleScene[0]); } catch (e) { Toast.error(e); } finally { diff --git a/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx b/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx index 86989d358..430758f71 100644 --- a/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx +++ b/ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx @@ -48,8 +48,8 @@ function renderScrapedStudioRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newStudio?: GQL.ScrapedSceneStudio, - onCreateNew?: (value: GQL.ScrapedSceneStudio) => void + newStudio?: GQL.ScrapedStudio, + onCreateNew?: (value: GQL.ScrapedStudio) => void ) { return ( , onChange: (value: ScrapeResult) => void, - newPerformers: GQL.ScrapedScenePerformer[], - onCreateNew?: (value: GQL.ScrapedScenePerformer) => void + newPerformers: GQL.ScrapedPerformer[], + onCreateNew?: (value: GQL.ScrapedPerformer) => void ) { + const performersCopy = newPerformers.map((p) => { + const name: string = p.name ?? ""; + return { ...p, name }; + }); + return ( ); @@ -142,9 +147,14 @@ function renderScrapedMoviesRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newMovies: GQL.ScrapedSceneMovie[], - onCreateNew?: (value: GQL.ScrapedSceneMovie) => void + newMovies: GQL.ScrapedMovie[], + onCreateNew?: (value: GQL.ScrapedMovie) => void ) { + const moviesCopy = newMovies.map((p) => { + const name: string = p.name ?? ""; + return { ...p, name }; + }); + return ( ); @@ -189,8 +199,8 @@ function renderScrapedTagsRow( title: string, result: ScrapeResult, onChange: (value: ScrapeResult) => void, - newTags: GQL.ScrapedSceneTag[], - onCreateNew?: (value: GQL.ScrapedSceneTag) => void + newTags: GQL.ScrapedTag[], + onCreateNew?: (value: GQL.ScrapedTag) => void ) { return ( = ( props.scraped.studio?.stored_id ) ); - const [newStudio, setNewStudio] = useState< - GQL.ScrapedSceneStudio | undefined - >( + const [newStudio, setNewStudio] = useState( props.scraped.studio && !props.scraped.studio.stored_id ? props.scraped.studio : undefined @@ -290,9 +298,9 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.performers ?? undefined) ) ); - const [newPerformers, setNewPerformers] = useState< - GQL.ScrapedScenePerformer[] - >(props.scraped.performers?.filter((t) => !t.stored_id) ?? []); + const [newPerformers, setNewPerformers] = useState( + props.scraped.performers?.filter((t) => !t.stored_id) ?? [] + ); const [movies, setMovies] = useState>( new ScrapeResult( @@ -300,7 +308,7 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.movies ?? undefined) ) ); - const [newMovies, setNewMovies] = useState( + const [newMovies, setNewMovies] = useState( props.scraped.movies?.filter((t) => !t.stored_id) ?? [] ); @@ -310,7 +318,7 @@ export const SceneScrapeDialog: React.FC = ( mapStoredIdObjects(props.scraped.tags ?? undefined) ) ); - const [newTags, setNewTags] = useState( + const [newTags, setNewTags] = useState( props.scraped.tags?.filter((t) => !t.stored_id) ?? [] ); @@ -339,7 +347,7 @@ export const SceneScrapeDialog: React.FC = ( return <>; } - async function createNewStudio(toCreate: GQL.ScrapedSceneStudio) { + async function createNewStudio(toCreate: GQL.ScrapedStudio) { try { const result = await createStudio({ variables: { @@ -366,7 +374,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewPerformer(toCreate: GQL.ScrapedScenePerformer) { + async function createNewPerformer(toCreate: GQL.ScrapedPerformer) { const input = makePerformerCreateInput(toCreate); try { @@ -401,7 +409,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewMovie(toCreate: GQL.ScrapedSceneMovie) { + async function createNewMovie(toCreate: GQL.ScrapedMovie) { let movieInput: GQL.MovieCreateInput = { name: "" }; try { movieInput = Object.assign(movieInput, toCreate); @@ -450,7 +458,7 @@ export const SceneScrapeDialog: React.FC = ( } } - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ diff --git a/ui/v2.5/src/components/Tagger/StashSearchResult.tsx b/ui/v2.5/src/components/Tagger/StashSearchResult.tsx index d370f2bd8..46c1fdc6b 100755 --- a/ui/v2.5/src/components/Tagger/StashSearchResult.tsx +++ b/ui/v2.5/src/components/Tagger/StashSearchResult.tsx @@ -105,7 +105,7 @@ interface IStashSearchResultProps { setTags: boolean; endpoint: string; queueFingerprintSubmission: (sceneId: string, endpoint: string) => void; - createNewTag: (toCreate: GQL.ScrapedSceneTag) => void; + createNewTag: (toCreate: GQL.ScrapedTag) => void; excludedFields: Record; setExcludedFields: (v: Record) => void; } diff --git a/ui/v2.5/src/components/Tagger/TaggerList.tsx b/ui/v2.5/src/components/Tagger/TaggerList.tsx index 7860c6809..5edba84b0 100644 --- a/ui/v2.5/src/components/Tagger/TaggerList.tsx +++ b/ui/v2.5/src/components/Tagger/TaggerList.tsx @@ -8,7 +8,6 @@ import { stashBoxSceneBatchQuery, useTagCreate } from "src/core/StashService"; import { SceneQueue } from "src/models/sceneQueue"; import { useToast } from "src/hooks"; -import { uniqBy } from "lodash"; import { ITaggerConfig } from "./constants"; import { selectScenes, IStashBoxScene } from "./utils"; import { TaggerScene } from "./TaggerScene"; @@ -25,7 +24,7 @@ interface ITaggerListProps { queue?: SceneQueue; selectedEndpoint: { endpoint: string; index: number }; config: ITaggerConfig; - queryScene: (searchVal: string) => Promise; + queryScene: (searchVal: string) => Promise; fingerprintQueue: IFingerprintQueue; } @@ -42,27 +41,8 @@ function fingerprintSearchResults( return ret; } - // perform matching here - scenes.forEach((scene) => { - // ignore where scene entry is not in results - if ( - (scene.checksum && fingerprints[scene.checksum] !== undefined) || - (scene.oshash && fingerprints[scene.oshash] !== undefined) || - (scene.phash && fingerprints[scene.phash] !== undefined) - ) { - const fingerprintMatches = uniqBy( - [ - ...(fingerprints[scene.checksum ?? ""] ?? []), - ...(fingerprints[scene.oshash ?? ""] ?? []), - ...(fingerprints[scene.phash ?? ""] ?? []), - ].flat(), - (f) => f.stash_id - ); - - ret[scene.id] = fingerprintMatches; - } else { - delete ret[scene.id]; - } + scenes.forEach((s) => { + ret[s.id] = fingerprints[s.id]; }); return ret; @@ -119,7 +99,7 @@ export const TaggerList: React.FC = ({ queryScene(searchVal) .then((queryData) => { - const s = selectScenes(queryData.queryStashBoxScene); + const s = selectScenes(queryData.scrapeSingleScene); setSearchResults({ ...searchResults, [sceneID]: s, @@ -179,26 +159,10 @@ export const TaggerList: React.FC = ({ // clear search errors setSearchErrors({}); - selectScenes(results.data?.queryStashBoxScene).forEach((scene) => { - scene.fingerprints?.forEach((f) => { - newFingerprints[f.hash] = newFingerprints[f.hash] - ? [...newFingerprints[f.hash], scene] - : [scene]; - }); - }); - - // Null any ids that are still undefined since it means they weren't found - filteredScenes.forEach((scene) => { - if (scene.oshash) { - newFingerprints[scene.oshash] = newFingerprints[scene.oshash] ?? null; - } - if (scene.checksum) { - newFingerprints[scene.checksum] = - newFingerprints[scene.checksum] ?? null; - } - if (scene.phash) { - newFingerprints[scene.phash] = newFingerprints[scene.phash] ?? null; - } + sceneIDs.forEach((sceneID, index) => { + newFingerprints[sceneID] = selectScenes( + results.data.scrapeMultiScenes[index] + ); }); const newSearchResults = fingerprintSearchResults(scenes, newFingerprints); @@ -210,7 +174,7 @@ export const TaggerList: React.FC = ({ setFingerprintError(""); }; - async function createNewTag(toCreate: GQL.ScrapedSceneTag) { + async function createNewTag(toCreate: GQL.ScrapedTag) { const tagInput: GQL.TagCreateInput = { name: toCreate.name ?? "" }; try { const result = await createTag({ @@ -259,20 +223,12 @@ export const TaggerList: React.FC = ({ const canFingerprintSearch = () => scenes.some( - (s) => - s.stash_ids.length === 0 && - (!s.oshash || fingerprints[s.oshash] === undefined) && - (!s.checksum || fingerprints[s.checksum] === undefined) && - (!s.phash || fingerprints[s.phash] === undefined) + (s) => s.stash_ids.length === 0 && fingerprints[s.id] === undefined ); const getFingerprintCount = () => { return scenes.filter( - (s) => - s.stash_ids.length === 0 && - ((s.checksum && fingerprints[s.checksum]) || - (s.oshash && fingerprints[s.oshash]) || - (s.phash && fingerprints[s.phash])) + (s) => s.stash_ids.length === 0 && fingerprints[s.id]?.length > 0 ).length; }; diff --git a/ui/v2.5/src/components/Tagger/TaggerScene.tsx b/ui/v2.5/src/components/Tagger/TaggerScene.tsx index d1f745780..3f4019522 100644 --- a/ui/v2.5/src/components/Tagger/TaggerScene.tsx +++ b/ui/v2.5/src/components/Tagger/TaggerScene.tsx @@ -95,7 +95,7 @@ export interface ITaggerScene { tagScene: (scene: Partial) => void; endpoint: string; queueFingerprintSubmission: (sceneId: string, endpoint: string) => void; - createNewTag: (toCreate: GQL.ScrapedSceneTag) => void; + createNewTag: (toCreate: GQL.ScrapedTag) => void; } export const TaggerScene: React.FC = ({ diff --git a/ui/v2.5/src/components/Tagger/performers/PerformerTagger.tsx b/ui/v2.5/src/components/Tagger/performers/PerformerTagger.tsx index 3ebb581e0..82e9cc622 100755 --- a/ui/v2.5/src/components/Tagger/performers/PerformerTagger.tsx +++ b/ui/v2.5/src/components/Tagger/performers/PerformerTagger.tsx @@ -97,9 +97,7 @@ const PerformerTaggerList: React.FC = ({ const doBoxSearch = (performerID: string, searchVal: string) => { stashBoxPerformerQuery(searchVal, selectedEndpoint.index) .then((queryData) => { - const s = selectPerformers( - queryData.data?.queryStashBoxPerformer?.[0].results ?? [] - ); + const s = selectPerformers(queryData.data?.scrapeSinglePerformer ?? []); setSearchResults({ ...searchResults, [performerID]: s, @@ -137,7 +135,7 @@ const PerformerTaggerList: React.FC = ({ stashBoxPerformerQuery(stashID, endpointIndex) .then((queryData) => { const data = selectPerformers( - queryData.data?.queryStashBoxPerformer?.[0].results ?? [] + queryData.data?.scrapeSinglePerformer ?? [] ); if (data.length > 0) { setModalPerformer({ diff --git a/ui/v2.5/src/components/Tagger/utils.ts b/ui/v2.5/src/components/Tagger/utils.ts index d403e1156..8ba60a5d5 100644 --- a/ui/v2.5/src/components/Tagger/utils.ts +++ b/ui/v2.5/src/components/Tagger/utils.ts @@ -201,7 +201,7 @@ export interface IStashBoxScene { fingerprints: IStashBoxFingerprint[]; } -const selectStudio = (studio: GQL.ScrapedSceneStudio): IStashBoxStudio => ({ +const selectStudio = (studio: GQL.ScrapedStudio): IStashBoxStudio => ({ id: studio?.stored_id ?? undefined, stash_id: studio.remote_site_id!, name: studio.name, @@ -212,14 +212,14 @@ const selectFingerprints = ( scene: GQL.ScrapedScene | null ): IStashBoxFingerprint[] => scene?.fingerprints ?? []; -const selectTags = (tags: GQL.ScrapedSceneTag[]): IStashBoxTag[] => +const selectTags = (tags: GQL.ScrapedTag[]): IStashBoxTag[] => tags.map((t) => ({ id: t.stored_id ?? undefined, name: t.name ?? "", })); export const selectPerformers = ( - performers: GQL.ScrapedScenePerformer[] + performers: GQL.ScrapedPerformer[] ): IStashBoxPerformer[] => performers.map((p) => ({ id: p.stored_id ?? undefined, diff --git a/ui/v2.5/src/core/StashService.ts b/ui/v2.5/src/core/StashService.ts index 1762a6ff9..a1bc343dd 100644 --- a/ui/v2.5/src/core/StashService.ts +++ b/ui/v2.5/src/core/StashService.ts @@ -257,17 +257,17 @@ export const useSceneMarkerDestroy = () => export const useListPerformerScrapers = () => GQL.useListPerformerScrapersQuery(); export const useScrapePerformerList = (scraperId: string, q: string) => - GQL.useScrapePerformerListQuery({ - variables: { scraper_id: scraperId, query: q }, + GQL.useScrapeSinglePerformerQuery({ + variables: { + source: { + scraper_id: scraperId, + }, + input: { + query: q, + }, + }, skip: q === "", }); -export const useScrapePerformer = ( - scraperId: string, - scrapedPerformer: GQL.ScrapedPerformerInput -) => - GQL.useScrapePerformerQuery({ - variables: { scraper_id: scraperId, scraped_performer: scrapedPerformer }, - }); export const useListSceneScrapers = () => GQL.useListSceneScrapersQuery(); @@ -814,11 +814,15 @@ export const queryScrapePerformer = ( scraperId: string, scrapedPerformer: GQL.ScrapedPerformerInput ) => - client.query({ - query: GQL.ScrapePerformerDocument, + client.query({ + query: GQL.ScrapeSinglePerformerDocument, variables: { - scraper_id: scraperId, - scraped_performer: scrapedPerformer, + source: { + scraper_id: scraperId, + }, + input: { + performer_input: scrapedPerformer, + }, }, fetchPolicy: "network-only", }); @@ -859,26 +863,29 @@ export const queryScrapeMovieURL = (url: string) => fetchPolicy: "network-only", }); -export const queryScrapeScene = ( - scraperId: string, - scene: GQL.SceneUpdateInput -) => - client.query({ - query: GQL.ScrapeSceneDocument, +export const queryScrapeScene = (scraperId: string, sceneId: string) => + client.query({ + query: GQL.ScrapeSingleSceneDocument, variables: { - scraper_id: scraperId, - scene, + source: { + scraper_id: scraperId, + }, + input: { + scene_id: sceneId, + }, }, fetchPolicy: "network-only", }); export const queryStashBoxScene = (stashBoxIndex: number, sceneID: string) => - client.query({ - query: GQL.QueryStashBoxSceneDocument, + client.query({ + query: GQL.ScrapeSingleSceneDocument, variables: { - input: { + source: { stash_box_index: stashBoxIndex, - scene_ids: [sceneID], + }, + input: { + scene_id: sceneID, }, }, }); @@ -887,25 +894,28 @@ export const queryStashBoxPerformer = ( stashBoxIndex: number, performerID: string ) => - client.query({ - query: GQL.QueryStashBoxPerformerDocument, + client.query({ + query: GQL.ScrapeSinglePerformerDocument, variables: { - input: { + source: { stash_box_index: stashBoxIndex, - performer_ids: [performerID], + }, + input: { + performer_id: performerID, }, }, }); -export const queryScrapeGallery = ( - scraperId: string, - gallery: GQL.GalleryUpdateInput -) => - client.query({ - query: GQL.ScrapeGalleryDocument, +export const queryScrapeGallery = (scraperId: string, galleryId: string) => + client.query({ + query: GQL.ScrapeSingleGalleryDocument, variables: { - scraper_id: scraperId, - gallery, + source: { + scraper_id: scraperId, + }, + input: { + gallery_id: galleryId, + }, }, fetchPolicy: "network-only", }); @@ -1017,11 +1027,9 @@ export const queryParseSceneFilenames = ( fetchPolicy: "network-only", }); -export const makePerformerCreateInput = ( - toCreate: GQL.ScrapedScenePerformer -) => { +export const makePerformerCreateInput = (toCreate: GQL.ScrapedPerformer) => { const input: GQL.PerformerCreateInput = { - name: toCreate.name, + name: toCreate.name ?? "", url: toCreate.url, gender: stringToGender(toCreate.gender), birthdate: toCreate.birthdate, @@ -1051,37 +1059,47 @@ export const makePerformerCreateInput = ( }; export const stashBoxSceneQuery = (searchVal: string, stashBoxIndex: number) => - client?.query< - GQL.QueryStashBoxSceneQuery, - GQL.QueryStashBoxSceneQueryVariables - >({ - query: GQL.QueryStashBoxSceneDocument, - variables: { input: { q: searchVal, stash_box_index: stashBoxIndex } }, + client.query({ + query: GQL.ScrapeSingleSceneDocument, + variables: { + source: { + stash_box_index: stashBoxIndex, + }, + input: { + query: searchVal, + }, + }, }); export const stashBoxPerformerQuery = ( searchVal: string, stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxPerformerQuery, - GQL.QueryStashBoxPerformerQueryVariables - >({ - query: GQL.QueryStashBoxPerformerDocument, - variables: { input: { q: searchVal, stash_box_index: stashBoxIndex } }, + client.query({ + query: GQL.ScrapeSinglePerformerDocument, + variables: { + source: { + stash_box_index: stashBoxIndex, + }, + input: { + query: searchVal, + }, + }, }); export const stashBoxSceneBatchQuery = ( sceneIds: string[], stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxSceneQuery, - GQL.QueryStashBoxSceneQueryVariables - >({ - query: GQL.QueryStashBoxSceneDocument, + client.query({ + query: GQL.ScrapeMultiScenesDocument, variables: { - input: { scene_ids: sceneIds, stash_box_index: stashBoxIndex }, + source: { + stash_box_index: stashBoxIndex, + }, + input: { + scene_ids: sceneIds, + }, }, }); @@ -1089,12 +1107,14 @@ export const stashBoxPerformerBatchQuery = ( performerIds: string[], stashBoxIndex: number ) => - client?.query< - GQL.QueryStashBoxPerformerQuery, - GQL.QueryStashBoxPerformerQueryVariables - >({ - query: GQL.QueryStashBoxPerformerDocument, + client.query({ + query: GQL.ScrapeMultiPerformersDocument, variables: { - input: { performer_ids: performerIds, stash_box_index: stashBoxIndex }, + source: { + stash_box_index: stashBoxIndex, + }, + input: { + performer_ids: performerIds, + }, }, });