Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update stash-box fingerprint query to fully support distance matching (#2509)

Browse Source
This commit is contained in:
InfiniteTF
2022-06-01 04:59:06 +02:00
committed by GitHub
parent 49f579e08e
commit e51083c26d
7 changed files with 466 additions and 444 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
graphql/schema/schema.graphql
View File

@@ -114,12 +114,6 @@ type Query {
"""Scrape a list of performers from a query"""
scrapeFreeonesPerformerList(query: String!): [String!]! @deprecated(reason: "use scrapeSinglePerformer with scraper_id = builtin_freeones")
"""Query StashBox for scenes"""
queryStashBoxScene(input: StashBoxSceneQueryInput!): [ScrapedScene!]! @deprecated(reason: "use scrapeSingleScene or scrapeMultiScenes")
"""Query StashBox for performers"""
queryStashBoxPerformer(input: StashBoxPerformerQueryInput!): [StashBoxPerformerQueryResult!]! @deprecated(reason: "use scrapeSinglePerformer or scrapeMultiPerformers")
# === end deprecated methods ===
# Plugins
"""List loaded plugins"""
plugins: [Plugin!]

6
graphql/stash-box/query.graphql
View File

@@ -129,6 +129,12 @@ query FindScenesByFullFingerprints($fingerprints: [FingerprintQueryInput!]!) {
}
}
query FindScenesBySceneFingerprints($fingerprints: [[FingerprintQueryInput!]!]!) {
findScenesBySceneFingerprints(fingerprints: $fingerprints) {
...SceneFragment
}
}
query SearchScene($term: String!) {
searchScene(term: $term) {
...SceneFragment

63
internal/api/resolver_query_scraper.go
View File

@@ -227,46 +227,6 @@ func (r *queryResolver) ScrapeMovieURL(ctx context.Context, url string) (*models
return marshalScrapedMovie(content)
}
func (r *queryResolver) QueryStashBoxScene(ctx context.Context, input models.StashBoxSceneQueryInput) ([]*models.ScrapedScene, error) {
boxes := config.GetInstance().GetStashBoxes()
if input.StashBoxIndex < 0 || input.StashBoxIndex >= len(boxes) {
return nil, fmt.Errorf("%w: invalid stash_box_index %d", ErrInput, input.StashBoxIndex)
}
client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager)
if len(input.SceneIds) > 0 {
return client.FindStashBoxScenesByFingerprintsFlat(ctx, input.SceneIds)
}
if input.Q != nil {
return client.QueryStashBoxScene(ctx, *input.Q)
}
return nil, nil
}
func (r *queryResolver) QueryStashBoxPerformer(ctx context.Context, input models.StashBoxPerformerQueryInput) ([]*models.StashBoxPerformerQueryResult, error) {
boxes := config.GetInstance().GetStashBoxes()
if input.StashBoxIndex < 0 || input.StashBoxIndex >= len(boxes) {
return nil, fmt.Errorf("%w: invalid stash_box_index %d", ErrInput, input.StashBoxIndex)
}
client := stashbox.NewClient(*boxes[input.StashBoxIndex], r.txnManager)
if len(input.PerformerIds) > 0 {
return client.FindStashBoxPerformersByNames(ctx, input.PerformerIds)
}
if input.Q != nil {
return client.QueryStashBoxPerformer(ctx, *input.Q)
}
return nil, nil
}
func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) {
boxes := config.GetInstance().GetStashBoxes()
@@ -280,6 +240,15 @@ func (r *queryResolver) getStashBoxClient(index int) (*stashbox.Client, error) {
func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.ScraperSourceInput, input models.ScrapeSingleSceneInput) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
var sceneID int
if input.SceneID != nil {
var err error
sceneID, err = strconv.Atoi(*input.SceneID)
if err != nil {
return nil, fmt.Errorf("%w: sceneID is not an integer: '%s'", ErrInput, *input.SceneID)
}
}
switch {
case source.ScraperID != nil:
var err error
@@ -288,11 +257,6 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr
switch {
case input.SceneID != nil:
var sceneID int
sceneID, err = strconv.Atoi(*input.SceneID)
if err != nil {
return nil, fmt.Errorf("%w: sceneID is not an integer: '%s'", ErrInput, *input.SceneID)
}
c, err = r.scraperCache().ScrapeID(ctx, *source.ScraperID, sceneID, models.ScrapeContentTypeScene)
if c != nil {
content = []models.ScrapedContent{c}
@@ -324,7 +288,7 @@ func (r *queryResolver) ScrapeSingleScene(ctx context.Context, source models.Scr
switch {
case input.SceneID != nil:
ret, err = client.FindStashBoxScenesByFingerprintsFlat(ctx, []string{*input.SceneID})
ret, err = client.FindStashBoxSceneByFingerprints(ctx, sceneID)
case input.Query != nil:
ret, err = client.QueryStashBoxScene(ctx, *input.Query)
default:
@@ -352,7 +316,12 @@ func (r *queryResolver) ScrapeMultiScenes(ctx context.Context, source models.Scr
return nil, err
}
return client.FindStashBoxScenesByFingerprints(ctx, input.SceneIds)
sceneIDs, err := stringslice.StringSliceToIntSlice(input.SceneIds)
if err != nil {
return nil, err
}
return client.FindStashBoxScenesByFingerprints(ctx, sceneIDs)
}
return nil, errors.New("scraper_id or stash_box_index must be set")

3
internal/manager/task_identify.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"errors"
"fmt"
"strconv"
"github.com/stashapp/stash/internal/identify"
"github.com/stashapp/stash/pkg/job"
@@ -212,7 +211,7 @@ type stashboxSource struct {
}
func (s stashboxSource) ScrapeScene(ctx context.Context, sceneID int) (*models.ScrapedScene, error) {
results, err := s.FindStashBoxScenesByFingerprintsFlat(ctx, []string{strconv.Itoa(sceneID)})
results, err := s.FindStashBoxSceneByFingerprints(ctx, sceneID)
if err != nil {
return nil, fmt.Errorf("error querying stash-box using scene ID %d: %w", sceneID, err)
}

495
pkg/scraper/stashbox/graphql/generated_client.go
View File

@@ -12,6 +12,7 @@ import (
type StashBoxGraphQLClient interface {
FindSceneByFingerprint(ctx context.Context, fingerprint FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByFingerprint, error)
FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error)
FindScenesBySceneFingerprints(ctx context.Context, fingerprints [][]*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesBySceneFingerprints, error)
SearchScene(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchScene, error)
SearchPerformer(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchPerformer, error)
FindPerformerByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindPerformerByID, error)
@@ -43,6 +44,7 @@ type Query struct {
FindSceneByFingerprint []*Scene "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\""
FindScenesByFingerprints []*Scene "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\""
FindScenesByFullFingerprints []*Scene "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\""
FindScenesBySceneFingerprints [][]*Scene "json:\"findScenesBySceneFingerprints\" graphql:\"findScenesBySceneFingerprints\""
QueryScenes QueryScenesResultType "json:\"queryScenes\" graphql:\"queryScenes\""
FindSite *Site "json:\"findSite\" graphql:\"findSite\""
QuerySites QuerySitesResultType "json:\"querySites\" graphql:\"querySites\""
@@ -95,6 +97,10 @@ type Mutation struct {
PerformerEdit Edit "json:\"performerEdit\" graphql:\"performerEdit\""
StudioEdit Edit "json:\"studioEdit\" graphql:\"studioEdit\""
TagEdit Edit "json:\"tagEdit\" graphql:\"tagEdit\""
SceneEditUpdate Edit "json:\"sceneEditUpdate\" graphql:\"sceneEditUpdate\""
PerformerEditUpdate Edit "json:\"performerEditUpdate\" graphql:\"performerEditUpdate\""
StudioEditUpdate Edit "json:\"studioEditUpdate\" graphql:\"studioEditUpdate\""
TagEditUpdate Edit "json:\"tagEditUpdate\" graphql:\"tagEditUpdate\""
EditVote Edit "json:\"editVote\" graphql:\"editVote\""
EditComment Edit "json:\"editComment\" graphql:\"editComment\""
ApplyEdit Edit "json:\"applyEdit\" graphql:\"applyEdit\""
@@ -190,6 +196,9 @@ type FindSceneByFingerprint struct {
type FindScenesByFullFingerprints struct {
FindScenesByFullFingerprints []*SceneFragment "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\""
}
type FindScenesBySceneFingerprints struct {
FindScenesBySceneFingerprints [][]*SceneFragment "json:\"findScenesBySceneFingerprints\" graphql:\"findScenesBySceneFingerprints\""
}
type SearchScene struct {
SearchScene []*SceneFragment "json:\"searchScene\" graphql:\"searchScene\""
}
@@ -240,6 +249,10 @@ fragment StudioFragment on Studio {
... ImageFragment
}
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerFragment on Performer {
id
name
@@ -274,16 +287,6 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
@@ -324,16 +327,22 @@ fragment ImageFragment on Image {
width
height
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
`
func (c *Client) FindSceneByFingerprint(ctx context.Context, fingerprint FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByFingerprint, error) {
@@ -354,10 +363,6 @@ const FindScenesByFullFingerprintsDocument = `query FindScenesByFullFingerprints
... SceneFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment StudioFragment on Studio {
name
id
@@ -368,11 +373,9 @@ fragment StudioFragment on Studio {
... ImageFragment
}
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerFragment on Performer {
id
@@ -412,6 +415,15 @@ fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment SceneFragment on Scene {
id
title
@@ -437,9 +449,81 @@ fragment SceneFragment on Scene {
... FingerprintFragment
}
}
fragment TagFragment on Tag {
name
fragment ImageFragment on Image {
id
url
width
height
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment URLFragment on URL {
url
type
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
`
func (c *Client) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) {
vars := map[string]interface{}{
"fingerprints": fingerprints,
}
var res FindScenesByFullFingerprints
if err := c.Client.Post(ctx, "FindScenesByFullFingerprints", FindScenesByFullFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil {
return nil, err
}
return &res, nil
}
const FindScenesBySceneFingerprintsDocument = `query FindScenesBySceneFingerprints ($fingerprints: [[FingerprintQueryInput!]!]!) {
findScenesBySceneFingerprints(fingerprints: $fingerprints) {
... SceneFragment
}
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment MeasurementsFragment on Measurements {
band_size
@@ -462,15 +546,68 @@ fragment ImageFragment on Image {
width
height
}
fragment URLFragment on URL {
url
type
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment SceneFragment on Scene {
id
title
details
duration
date
urls {
... URLFragment
}
images {
... ImageFragment
}
studio {
... StudioFragment
}
tags {
... TagFragment
}
performers {
... PerformerAppearanceFragment
}
fingerprints {
... FingerprintFragment
}
}
`
func (c *Client) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) {
func (c *Client) FindScenesBySceneFingerprints(ctx context.Context, fingerprints [][]*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesBySceneFingerprints, error) {
vars := map[string]interface{}{
"fingerprints": fingerprints,
}
var res FindScenesByFullFingerprints
if err := c.Client.Post(ctx, "FindScenesByFullFingerprints", FindScenesByFullFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil {
var res FindScenesBySceneFingerprints
if err := c.Client.Post(ctx, "FindScenesBySceneFingerprints", FindScenesBySceneFingerprintsDocument, &res, vars, httpRequestOptions...); err != nil {
return nil, err
}
@@ -490,72 +627,6 @@ fragment TagFragment on Tag {
name
id
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment SceneFragment on Scene {
id
title
details
duration
date
urls {
... URLFragment
}
images {
... ImageFragment
}
studio {
... StudioFragment
}
tags {
... TagFragment
}
performers {
... PerformerAppearanceFragment
}
fingerprints {
... FingerprintFragment
}
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment PerformerFragment on Performer {
id
name
@@ -590,6 +661,72 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment SceneFragment on Scene {
id
title
details
duration
date
urls {
... URLFragment
}
images {
... ImageFragment
}
studio {
... StudioFragment
}
tags {
... TagFragment
}
performers {
... PerformerAppearanceFragment
}
fingerprints {
... FingerprintFragment
}
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment ImageFragment on Image {
id
url
width
height
}
`
func (c *Client) SearchScene(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchScene, error) {
@@ -610,16 +747,6 @@ const SearchPerformerDocument = `query SearchPerformer ($term: String!) {
... PerformerFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
@@ -668,6 +795,16 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment ImageFragment on Image {
id
url
width
height
}
`
func (c *Client) SearchPerformer(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchPerformer, error) {
@@ -688,6 +825,16 @@ const FindPerformerByIDDocument = `query FindPerformerByID ($id: ID!) {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
@@ -736,16 +883,6 @@ fragment ImageFragment on Image {
width
height
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
`
func (c *Client) FindPerformerByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindPerformerByID, error) {
@@ -766,63 +903,11 @@ const FindSceneByIDDocument = `query FindSceneByID ($id: ID!) {
... SceneFragment
}
}
fragment PerformerFragment on Performer {
fragment ImageFragment on Image {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment URLFragment on URL {
url
type
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
width
height
}
fragment TagFragment on Tag {
name
@@ -834,14 +919,15 @@ fragment PerformerAppearanceFragment on PerformerAppearance {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment SceneFragment on Scene {
id
@@ -868,11 +954,62 @@ fragment SceneFragment on Scene {
... FingerprintFragment
}
}
fragment ImageFragment on Image {
id
fragment URLFragment on URL {
url
width
type
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
`

78
pkg/scraper/stashbox/graphql/generated_models.go
View File

@@ -88,8 +88,8 @@ type DraftEntity struct {
ID *string `json:"id,omitempty"`
}
func (DraftEntity) IsSceneDraftPerformer() {}
func (DraftEntity) IsSceneDraftStudio() {}
func (DraftEntity) IsSceneDraftPerformer() {}
func (DraftEntity) IsSceneDraftTag() {}
type DraftEntityInput struct {
@@ -130,7 +130,9 @@ type Edit struct {
Status VoteStatusEnum `json:"status"`
Applied bool `json:"applied"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
Updated *time.Time `json:"updated,omitempty"`
Closed *time.Time `json:"closed,omitempty"`
Expires *time.Time `json:"expires,omitempty"`
}
type EditComment struct {
@@ -149,8 +151,6 @@ type EditInput struct {
// Not required for create type
ID *string `json:"id,omitempty"`
Operation OperationEnum `json:"operation"`
// Required for amending an existing edit
EditID *string `json:"edit_id,omitempty"`
// Only required for merge type
MergeSourceIds []string `json:"merge_source_ids,omitempty"`
Comment *string `json:"comment,omitempty"`
@@ -211,9 +211,7 @@ type FingerprintEditInput struct {
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
Created time.Time `json:"created"`
// @deprecated(reason: "unused")
Submissions *int `json:"submissions,omitempty"`
// @deprecated(reason: "unused")
Updated *time.Time `json:"updated,omitempty"`
}
@@ -241,11 +239,6 @@ type FuzzyDate struct {
Accuracy DateAccuracyEnum `json:"accuracy"`
}
type FuzzyDateInput struct {
Date string `json:"date"`
Accuracy DateAccuracyEnum `json:"accuracy"`
}
type GrantInviteInput struct {
UserID string `json:"user_id"`
Amount int `json:"amount"`
@@ -294,13 +287,6 @@ type Measurements struct {
Hip *int `json:"hip,omitempty"`
}
type MeasurementsInput struct {
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
Waist *int `json:"waist,omitempty"`
Hip *int `json:"hip,omitempty"`
}
type MultiIDCriterionInput struct {
Value []string `json:"value,omitempty"`
Modifier CriterionModifier `json:"modifier"`
@@ -324,6 +310,7 @@ type Performer struct {
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URL `json:"urls,omitempty"`
Birthdate *FuzzyDate `json:"birthdate,omitempty"`
BirthDate *string `json:"birth_date,omitempty"`
Age *int `json:"age,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
@@ -332,6 +319,10 @@ type Performer struct {
// Height in cm
Height *int `json:"height,omitempty"`
Measurements *Measurements `json:"measurements,omitempty"`
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
WaistSize *int `json:"waist_size,omitempty"`
HipSize *int `json:"hip_size,omitempty"`
BreastType *BreastTypeEnum `json:"breast_type,omitempty"`
CareerStartYear *int `json:"career_start_year,omitempty"`
CareerEndYear *int `json:"career_end_year,omitempty"`
@@ -348,8 +339,8 @@ type Performer struct {
Updated time.Time `json:"updated"`
}
func (Performer) IsSceneDraftPerformer() {}
func (Performer) IsEditTarget() {}
func (Performer) IsSceneDraftPerformer() {}
type PerformerAppearance struct {
Performer *Performer `json:"performer,omitempty"`
@@ -369,13 +360,16 @@ type PerformerCreateInput struct {
Aliases []string `json:"aliases,omitempty"`
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *FuzzyDateInput `json:"birthdate,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
HairColor *HairColorEnum `json:"hair_color,omitempty"`
Height *int `json:"height,omitempty"`
Measurements *MeasurementsInput `json:"measurements,omitempty"`
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
WaistSize *int `json:"waist_size,omitempty"`
HipSize *int `json:"hip_size,omitempty"`
BreastType *BreastTypeEnum `json:"breast_type,omitempty"`
CareerStartYear *int `json:"career_start_year,omitempty"`
CareerEndYear *int `json:"career_end_year,omitempty"`
@@ -390,6 +384,7 @@ type PerformerDestroyInput struct {
}
type PerformerDraft struct {
ID *string `json:"id,omitempty"`
Name string `json:"name"`
Aliases *string `json:"aliases,omitempty"`
Gender *string `json:"gender,omitempty"`
@@ -412,6 +407,7 @@ type PerformerDraft struct {
func (PerformerDraft) IsDraftData() {}
type PerformerDraftInput struct {
ID *string `json:"id,omitempty"`
Name string `json:"name"`
Aliases *string `json:"aliases,omitempty"`
Gender *string `json:"gender,omitempty"`
@@ -440,7 +436,6 @@ type PerformerEdit struct {
AddedUrls []*URL `json:"added_urls,omitempty"`
RemovedUrls []*URL `json:"removed_urls,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
BirthdateAccuracy *string `json:"birthdate_accuracy,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
@@ -461,6 +456,11 @@ type PerformerEdit struct {
AddedImages []*Image `json:"added_images,omitempty"`
RemovedImages []*Image `json:"removed_images,omitempty"`
DraftID *string `json:"draft_id,omitempty"`
Aliases []string `json:"aliases,omitempty"`
Urls []*URL `json:"urls,omitempty"`
Images []*Image `json:"images,omitempty"`
Tattoos []*BodyModification `json:"tattoos,omitempty"`
Piercings []*BodyModification `json:"piercings,omitempty"`
}
func (PerformerEdit) IsEditDetails() {}
@@ -471,13 +471,16 @@ type PerformerEditDetailsInput struct {
Aliases []string `json:"aliases,omitempty"`
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *FuzzyDateInput `json:"birthdate,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
HairColor *HairColorEnum `json:"hair_color,omitempty"`
Height *int `json:"height,omitempty"`
Measurements *MeasurementsInput `json:"measurements,omitempty"`
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
WaistSize *int `json:"waist_size,omitempty"`
HipSize *int `json:"hip_size,omitempty"`
BreastType *BreastTypeEnum `json:"breast_type,omitempty"`
CareerStartYear *int `json:"career_start_year,omitempty"`
CareerEndYear *int `json:"career_end_year,omitempty"`
@@ -510,7 +513,7 @@ type PerformerEditOptionsInput struct {
}
type PerformerQueryInput struct {
// Searches name and aliases - assumes like query unless quoted
// Searches name and disambiguation - assumes like query unless quoted
Names *string `json:"names,omitempty"`
// Searches name only - assumes like query unless quoted
Name *string `json:"name,omitempty"`
@@ -557,13 +560,16 @@ type PerformerUpdateInput struct {
Aliases []string `json:"aliases,omitempty"`
Gender *GenderEnum `json:"gender,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Birthdate *FuzzyDateInput `json:"birthdate,omitempty"`
Birthdate *string `json:"birthdate,omitempty"`
Ethnicity *EthnicityEnum `json:"ethnicity,omitempty"`
Country *string `json:"country,omitempty"`
EyeColor *EyeColorEnum `json:"eye_color,omitempty"`
HairColor *HairColorEnum `json:"hair_color,omitempty"`
Height *int `json:"height,omitempty"`
Measurements *MeasurementsInput `json:"measurements,omitempty"`
CupSize *string `json:"cup_size,omitempty"`
BandSize *int `json:"band_size,omitempty"`
WaistSize *int `json:"waist_size,omitempty"`
HipSize *int `json:"hip_size,omitempty"`
BreastType *BreastTypeEnum `json:"breast_type,omitempty"`
CareerStartYear *int `json:"career_start_year,omitempty"`
CareerEndYear *int `json:"career_end_year,omitempty"`
@@ -631,6 +637,7 @@ type Scene struct {
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Date *string `json:"date,omitempty"`
ReleaseDate *string `json:"release_date,omitempty"`
Urls []*URL `json:"urls,omitempty"`
Studio *Studio `json:"studio,omitempty"`
Tags []*Tag `json:"tags,omitempty"`
@@ -652,7 +659,7 @@ type SceneCreateInput struct {
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
Urls []*URLInput `json:"urls,omitempty"`
Date *string `json:"date,omitempty"`
Date string `json:"date"`
StudioID *string `json:"studio_id,omitempty"`
Performers []*PerformerAppearanceInput `json:"performers,omitempty"`
TagIds []string `json:"tag_ids,omitempty"`
@@ -668,6 +675,7 @@ type SceneDestroyInput struct {
}
type SceneDraft struct {
ID *string `json:"id,omitempty"`
Title *string `json:"title,omitempty"`
Details *string `json:"details,omitempty"`
URL *URL `json:"url,omitempty"`
@@ -701,6 +709,11 @@ type SceneEdit struct {
Director *string `json:"director,omitempty"`
Code *string `json:"code,omitempty"`
DraftID *string `json:"draft_id,omitempty"`
Urls []*URL `json:"urls,omitempty"`
Performers []*PerformerAppearance `json:"performers,omitempty"`
Tags []*Tag `json:"tags,omitempty"`
Images []*Image `json:"images,omitempty"`
Fingerprints []*Fingerprint `json:"fingerprints,omitempty"`
}
func (SceneEdit) IsEditDetails() {}
@@ -855,6 +868,8 @@ type StudioEdit struct {
Parent *Studio `json:"parent,omitempty"`
AddedImages []*Image `json:"added_images,omitempty"`
RemovedImages []*Image `json:"removed_images,omitempty"`
Images []*Image `json:"images,omitempty"`
Urls []*URL `json:"urls,omitempty"`
}
func (StudioEdit) IsEditDetails() {}
@@ -909,8 +924,8 @@ type Tag struct {
Updated time.Time `json:"updated"`
}
func (Tag) IsSceneDraftTag() {}
func (Tag) IsEditTarget() {}
func (Tag) IsSceneDraftTag() {}
type TagCategory struct {
ID string `json:"id"`
@@ -953,6 +968,7 @@ type TagEdit struct {
AddedAliases []string `json:"added_aliases,omitempty"`
RemovedAliases []string `json:"removed_aliases,omitempty"`
Category *TagCategory `json:"category,omitempty"`
Aliases []string `json:"aliases,omitempty"`
}
func (TagEdit) IsEditDetails() {}
@@ -1256,16 +1272,18 @@ type EditSortEnum string
const (
EditSortEnumCreatedAt EditSortEnum = "CREATED_AT"
EditSortEnumUpdatedAt EditSortEnum = "UPDATED_AT"
EditSortEnumClosedAt EditSortEnum = "CLOSED_AT"
)
var AllEditSortEnum = []EditSortEnum{
EditSortEnumCreatedAt,
EditSortEnumUpdatedAt,
EditSortEnumClosedAt,
}
func (e EditSortEnum) IsValid() bool {
switch e {
case EditSortEnumCreatedAt, EditSortEnumUpdatedAt:
case EditSortEnumCreatedAt, EditSortEnumUpdatedAt, EditSortEnumClosedAt:
return true
}
return false

165
pkg/scraper/stashbox/stash_box.go
View File

@@ -14,7 +14,6 @@ import (
"strings"
"github.com/Yamashou/gqlgenc/client"
"github.com/corona10/goimagehash"
"golang.org/x/text/cases"
"golang.org/x/text/language"
@@ -24,7 +23,6 @@ import (
"github.com/stashapp/stash/pkg/match"
"github.com/stashapp/stash/pkg/models"
"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
"github.com/stashapp/stash/pkg/sliceutil/intslice"
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
"github.com/stashapp/stash/pkg/utils"
)
@@ -78,127 +76,21 @@ func (c Client) QueryStashBoxScene(ctx context.Context, queryStr string) ([]*mod
return ret, nil
}
func phashMatches(hash, other int64) bool {
// HACK - stash-box match distance is configurable. This needs to be fixed on
// the stash-box end.
const stashBoxDistance = 4
imageHash := goimagehash.NewImageHash(uint64(hash), goimagehash.PHash)
otherHash := goimagehash.NewImageHash(uint64(other), goimagehash.PHash)
distance, _ := imageHash.Distance(otherHash)
return distance <= stashBoxDistance
// FindStashBoxScenesByFingerprints queries stash-box for a scene using the
// scene's MD5/OSHASH checksum, or PHash.
func (c Client) FindStashBoxSceneByFingerprints(ctx context.Context, sceneID int) ([]*models.ScrapedScene, error) {
res, err := c.FindStashBoxScenesByFingerprints(ctx, []int{sceneID})
if len(res) > 0 {
return res[0], err
}
return nil, err
}
// FindStashBoxScenesByFingerprints queries stash-box for scenes using every
// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order
// as the input slice.
func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, sceneIDs []string) ([][]*models.ScrapedScene, error) {
ids, err := stringslice.StringSliceToIntSlice(sceneIDs)
if err != nil {
return nil, err
}
var fingerprints []*graphql.FingerprintQueryInput
// map fingerprints to their scene index
fpToScene := make(map[string][]int)
phashToScene := make(map[int64][]int)
if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
qb := r.Scene()
for index, sceneID := range ids {
scene, err := qb.Find(sceneID)
if err != nil {
return err
}
if scene == nil {
return fmt.Errorf("scene with id %d not found", sceneID)
}
if scene.Checksum.Valid {
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.Checksum.String,
Algorithm: graphql.FingerprintAlgorithmMd5,
})
fpToScene[scene.Checksum.String] = append(fpToScene[scene.Checksum.String], index)
}
if scene.OSHash.Valid {
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.OSHash.String,
Algorithm: graphql.FingerprintAlgorithmOshash,
})
fpToScene[scene.OSHash.String] = append(fpToScene[scene.OSHash.String], index)
}
if scene.Phash.Valid {
phashStr := utils.PhashToString(scene.Phash.Int64)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: phashStr,
Algorithm: graphql.FingerprintAlgorithmPhash,
})
fpToScene[phashStr] = append(fpToScene[phashStr], index)
phashToScene[scene.Phash.Int64] = append(phashToScene[scene.Phash.Int64], index)
}
}
return nil
}); err != nil {
return nil, err
}
allScenes, err := c.findStashBoxScenesByFingerprints(ctx, fingerprints)
if err != nil {
return nil, err
}
// set the matched scenes back in their original order
ret := make([][]*models.ScrapedScene, len(sceneIDs))
for _, s := range allScenes {
var addedTo []int
addScene := func(sceneIndexes []int) {
for _, index := range sceneIndexes {
if !intslice.IntInclude(addedTo, index) {
addedTo = append(addedTo, index)
ret[index] = append(ret[index], s)
}
}
}
for _, fp := range s.Fingerprints {
addScene(fpToScene[fp.Hash])
// HACK - we really need stash-box to return specific hash-to-result sets
if fp.Algorithm == graphql.FingerprintAlgorithmPhash.String() {
hash, err := utils.StringToPhash(fp.Hash)
if err != nil {
continue
}
for phash, sceneIndexes := range phashToScene {
if phashMatches(hash, phash) {
addScene(sceneIndexes)
}
}
}
}
}
return ret, nil
}
// FindStashBoxScenesByFingerprintsFlat queries stash-box for scenes using every
// scene's MD5/OSHASH checksum, or PHash, and returns results a flat slice.
func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneIDs []string) ([]*models.ScrapedScene, error) {
ids, err := stringslice.StringSliceToIntSlice(sceneIDs)
if err != nil {
return nil, err
}
var fingerprints []*graphql.FingerprintQueryInput
func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, ids []int) ([][]*models.ScrapedScene, error) {
var fingerprints [][]*graphql.FingerprintQueryInput
if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
qb := r.Scene()
@@ -213,26 +105,31 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneI
return fmt.Errorf("scene with id %d not found", sceneID)
}
var sceneFPs []*graphql.FingerprintQueryInput
if scene.Checksum.Valid {
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
Hash: scene.Checksum.String,
Algorithm: graphql.FingerprintAlgorithmMd5,
})
}
if scene.OSHash.Valid {
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
Hash: scene.OSHash.String,
Algorithm: graphql.FingerprintAlgorithmOshash,
})
}
if scene.Phash.Valid {
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: utils.PhashToString(scene.Phash.Int64),
phashStr := utils.PhashToString(scene.Phash.Int64)
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
Hash: phashStr,
Algorithm: graphql.FingerprintAlgorithmPhash,
})
}
fingerprints = append(fingerprints, sceneFPs)
}
return nil
@@ -243,27 +140,29 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(ctx context.Context, sceneI
return c.findStashBoxScenesByFingerprints(ctx, fingerprints)
}
func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprints []*graphql.FingerprintQueryInput) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
for i := 0; i < len(fingerprints); i += 100 {
func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, scenes [][]*graphql.FingerprintQueryInput) ([][]*models.ScrapedScene, error) {
var ret [][]*models.ScrapedScene
for i := 0; i < len(scenes); i += 40 {
end := i + 100
if end > len(fingerprints) {
end = len(fingerprints)
if end > len(scenes) {
end = len(scenes)
}
scenes, err := c.client.FindScenesByFullFingerprints(ctx, fingerprints[i:end])
scenes, err := c.client.FindScenesBySceneFingerprints(ctx, scenes[i:end])
if err != nil {
return nil, err
}
sceneFragments := scenes.FindScenesByFullFingerprints
for _, s := range sceneFragments {
ss, err := c.sceneFragmentToScrapedScene(ctx, s)
for _, sceneFragments := range scenes.FindScenesBySceneFingerprints {
var sceneResults []*models.ScrapedScene
for _, scene := range sceneFragments {
ss, err := c.sceneFragmentToScrapedScene(ctx, scene)
if err != nil {
return nil, err
}
ret = append(ret, ss)
sceneResults = append(sceneResults, ss)
}
ret = append(ret, sceneResults)
}
}