Add PHash distance matching to stash-box integration (#1858)

* Add PHash distance matching to stash-box integration
This commit is contained in:
InfiniteTF
2021-10-20 08:22:25 +02:00
committed by GitHub
parent 976038424b
commit 15acf91b90
21 changed files with 462 additions and 217 deletions

View File

@@ -18,26 +18,27 @@ func NewClient(cli *http.Client, baseURL string, options ...client.HTTPRequestOp
}
type Query struct {
FindPerformer *Performer "json:\"findPerformer\" graphql:\"findPerformer\""
QueryPerformers QueryPerformersResultType "json:\"queryPerformers\" graphql:\"queryPerformers\""
FindStudio *Studio "json:\"findStudio\" graphql:\"findStudio\""
QueryStudios QueryStudiosResultType "json:\"queryStudios\" graphql:\"queryStudios\""
FindTag *Tag "json:\"findTag\" graphql:\"findTag\""
QueryTags QueryTagsResultType "json:\"queryTags\" graphql:\"queryTags\""
FindTagCategory *TagCategory "json:\"findTagCategory\" graphql:\"findTagCategory\""
QueryTagCategories QueryTagCategoriesResultType "json:\"queryTagCategories\" graphql:\"queryTagCategories\""
FindScene *Scene "json:\"findScene\" graphql:\"findScene\""
FindSceneByFingerprint []*Scene "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\""
FindScenesByFingerprints []*Scene "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\""
QueryScenes QueryScenesResultType "json:\"queryScenes\" graphql:\"queryScenes\""
FindEdit *Edit "json:\"findEdit\" graphql:\"findEdit\""
QueryEdits QueryEditsResultType "json:\"queryEdits\" graphql:\"queryEdits\""
FindUser *User "json:\"findUser\" graphql:\"findUser\""
QueryUsers QueryUsersResultType "json:\"queryUsers\" graphql:\"queryUsers\""
Me *User "json:\"me\" graphql:\"me\""
SearchPerformer []*Performer "json:\"searchPerformer\" graphql:\"searchPerformer\""
SearchScene []*Scene "json:\"searchScene\" graphql:\"searchScene\""
Version Version "json:\"version\" graphql:\"version\""
FindPerformer *Performer "json:\"findPerformer\" graphql:\"findPerformer\""
QueryPerformers QueryPerformersResultType "json:\"queryPerformers\" graphql:\"queryPerformers\""
FindStudio *Studio "json:\"findStudio\" graphql:\"findStudio\""
QueryStudios QueryStudiosResultType "json:\"queryStudios\" graphql:\"queryStudios\""
FindTag *Tag "json:\"findTag\" graphql:\"findTag\""
QueryTags QueryTagsResultType "json:\"queryTags\" graphql:\"queryTags\""
FindTagCategory *TagCategory "json:\"findTagCategory\" graphql:\"findTagCategory\""
QueryTagCategories QueryTagCategoriesResultType "json:\"queryTagCategories\" graphql:\"queryTagCategories\""
FindScene *Scene "json:\"findScene\" graphql:\"findScene\""
FindSceneByFingerprint []*Scene "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\""
FindScenesByFingerprints []*Scene "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\""
FindScenesByFullFingerprints []*Scene "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\""
QueryScenes QueryScenesResultType "json:\"queryScenes\" graphql:\"queryScenes\""
FindEdit *Edit "json:\"findEdit\" graphql:\"findEdit\""
QueryEdits QueryEditsResultType "json:\"queryEdits\" graphql:\"queryEdits\""
FindUser *User "json:\"findUser\" graphql:\"findUser\""
QueryUsers QueryUsersResultType "json:\"queryUsers\" graphql:\"queryUsers\""
Me *User "json:\"me\" graphql:\"me\""
SearchPerformer []*Performer "json:\"searchPerformer\" graphql:\"searchPerformer\""
SearchScene []*Scene "json:\"searchScene\" graphql:\"searchScene\""
Version Version "json:\"version\" graphql:\"version\""
}
type Mutation struct {
@@ -120,6 +121,7 @@ type PerformerFragment struct {
Disambiguation *string "json:\"disambiguation\" graphql:\"disambiguation\""
Aliases []string "json:\"aliases\" graphql:\"aliases\""
Gender *GenderEnum "json:\"gender\" graphql:\"gender\""
MergedIds []string "json:\"merged_ids\" graphql:\"merged_ids\""
Urls []*URLFragment "json:\"urls\" graphql:\"urls\""
Images []*ImageFragment "json:\"images\" graphql:\"images\""
Birthdate *FuzzyDateFragment "json:\"birthdate\" graphql:\"birthdate\""
@@ -160,8 +162,8 @@ type SceneFragment struct {
type FindSceneByFingerprint struct {
FindSceneByFingerprint []*SceneFragment "json:\"findSceneByFingerprint\" graphql:\"findSceneByFingerprint\""
}
type FindScenesByFingerprints struct {
FindScenesByFingerprints []*SceneFragment "json:\"findScenesByFingerprints\" graphql:\"findScenesByFingerprints\""
type FindScenesByFullFingerprints struct {
FindScenesByFullFingerprints []*SceneFragment "json:\"findScenesByFullFingerprints\" graphql:\"findScenesByFullFingerprints\""
}
type SearchScene struct {
SearchScene []*SceneFragment "json:\"searchScene\" graphql:\"searchScene\""
@@ -172,6 +174,9 @@ type SearchPerformer struct {
type FindPerformerByID struct {
FindPerformer *PerformerFragment "json:\"findPerformer\" graphql:\"findPerformer\""
}
type FindSceneByID struct {
FindScene *SceneFragment "json:\"findScene\" graphql:\"findScene\""
}
type SubmitFingerprintPayload struct {
SubmitFingerprint bool "json:\"submitFingerprint\" graphql:\"submitFingerprint\""
}
@@ -181,56 +186,10 @@ const FindSceneByFingerprintQuery = `query FindSceneByFingerprint ($fingerprint:
... SceneFragment
}
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment SceneFragment on Scene {
id
title
@@ -270,24 +229,71 @@ fragment StudioFragment on Studio {
... ImageFragment
}
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment TagFragment on Tag {
name
id
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
`
func (c *Client) FindSceneByFingerprint(ctx context.Context, fingerprint FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByFingerprint, error) {
@@ -303,31 +309,38 @@ func (c *Client) FindSceneByFingerprint(ctx context.Context, fingerprint Fingerp
return &res, nil
}
const FindScenesByFingerprintsQuery = `query FindScenesByFingerprints ($fingerprints: [String!]!) {
findScenesByFingerprints(fingerprints: $fingerprints) {
const FindScenesByFullFingerprintsQuery = `query FindScenesByFullFingerprints ($fingerprints: [FingerprintQueryInput!]!) {
findScenesByFullFingerprints(fingerprints: $fingerprints) {
... SceneFragment
}
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment TagFragment on Tag {
name
id
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
@@ -355,16 +368,16 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment SceneFragment on Scene {
id
title
@@ -390,40 +403,34 @@ fragment SceneFragment on Scene {
... FingerprintFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment URLFragment on URL {
url
type
}
`
func (c *Client) FindScenesByFingerprints(ctx context.Context, fingerprints []string, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFingerprints, error) {
func (c *Client) FindScenesByFullFingerprints(ctx context.Context, fingerprints []*FingerprintQueryInput, httpRequestOptions ...client.HTTPRequestOption) (*FindScenesByFullFingerprints, error) {
vars := map[string]interface{}{
"fingerprints": fingerprints,
}
var res FindScenesByFingerprints
if err := c.Client.Post(ctx, FindScenesByFingerprintsQuery, &res, vars, httpRequestOptions...); err != nil {
var res FindScenesByFullFingerprints
if err := c.Client.Post(ctx, FindScenesByFullFingerprintsQuery, &res, vars, httpRequestOptions...); err != nil {
return nil, err
}
@@ -435,21 +442,6 @@ const SearchSceneQuery = `query SearchScene ($term: String!) {
... SceneFragment
}
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment URLFragment on URL {
url
type
@@ -468,6 +460,11 @@ fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment SceneFragment on Scene {
id
title
@@ -515,6 +512,7 @@ fragment PerformerFragment on Performer {
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
@@ -542,6 +540,16 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
`
func (c *Client) SearchScene(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchScene, error) {
@@ -562,6 +570,16 @@ const SearchPerformerQuery = `query SearchPerformer ($term: String!) {
... PerformerFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
@@ -582,6 +600,7 @@ fragment PerformerFragment on Performer {
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
@@ -609,16 +628,6 @@ fragment PerformerFragment on Performer {
... BodyModificationFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment ImageFragment on Image {
id
url
width
height
}
`
func (c *Client) SearchPerformer(ctx context.Context, term string, httpRequestOptions ...client.HTTPRequestOption) (*SearchPerformer, error) {
@@ -639,26 +648,13 @@ const FindPerformerByIDQuery = `query FindPerformerByID ($id: ID!) {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
@@ -696,6 +692,20 @@ fragment ImageFragment on Image {
width
height
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment BodyModificationFragment on BodyModification {
location
description
}
`
func (c *Client) FindPerformerByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindPerformerByID, error) {
@@ -711,6 +721,134 @@ func (c *Client) FindPerformerByID(ctx context.Context, id string, httpRequestOp
return &res, nil
}
const FindSceneByIDQuery = `query FindSceneByID ($id: ID!) {
findScene(id: $id) {
... SceneFragment
}
}
fragment ImageFragment on Image {
id
url
width
height
}
fragment StudioFragment on Studio {
name
id
urls {
... URLFragment
}
images {
... ImageFragment
}
}
fragment TagFragment on Tag {
name
id
}
fragment PerformerFragment on Performer {
id
name
disambiguation
aliases
gender
merged_ids
urls {
... URLFragment
}
images {
... ImageFragment
}
birthdate {
... FuzzyDateFragment
}
ethnicity
country
eye_color
hair_color
height
measurements {
... MeasurementsFragment
}
breast_type
career_start_year
career_end_year
tattoos {
... BodyModificationFragment
}
piercings {
... BodyModificationFragment
}
}
fragment MeasurementsFragment on Measurements {
band_size
cup_size
waist
hip
}
fragment FingerprintFragment on Fingerprint {
algorithm
hash
duration
}
fragment SceneFragment on Scene {
id
title
details
duration
date
urls {
... URLFragment
}
images {
... ImageFragment
}
studio {
... StudioFragment
}
tags {
... TagFragment
}
performers {
... PerformerAppearanceFragment
}
fingerprints {
... FingerprintFragment
}
}
fragment URLFragment on URL {
url
type
}
fragment BodyModificationFragment on BodyModification {
location
description
}
fragment PerformerAppearanceFragment on PerformerAppearance {
as
performer {
... PerformerFragment
}
}
fragment FuzzyDateFragment on FuzzyDate {
date
accuracy
}
`
func (c *Client) FindSceneByID(ctx context.Context, id string, httpRequestOptions ...client.HTTPRequestOption) (*FindSceneByID, error) {
vars := map[string]interface{}{
"id": id,
}
var res FindSceneByID
if err := c.Client.Post(ctx, FindSceneByIDQuery, &res, vars, httpRequestOptions...); err != nil {
return nil, err
}
return &res, nil
}
const SubmitFingerprintQuery = `mutation SubmitFingerprint ($input: FingerprintSubmission!) {
submitFingerprint(input: $input)
}

View File

@@ -81,6 +81,7 @@ type Edit struct {
Status VoteStatusEnum `json:"status"`
Applied bool `json:"applied"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
type EditComment struct {
@@ -134,9 +135,21 @@ type EyeColorCriterionInput struct {
}
type Fingerprint struct {
Hash string `json:"hash"`
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
Hash string `json:"hash"`
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
Submissions int `json:"submissions"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
type FingerprintEditInput struct {
Hash string `json:"hash"`
Algorithm FingerprintAlgorithm `json:"algorithm"`
Duration int `json:"duration"`
Submissions int `json:"submissions"`
Created time.Time `json:"created"`
Updated time.Time `json:"updated"`
}
type FingerprintInput struct {
@@ -255,6 +268,8 @@ type Performer struct {
Deleted bool `json:"deleted"`
Edits []*Edit `json:"edits"`
SceneCount int `json:"scene_count"`
MergedIds []string `json:"merged_ids"`
Studios []*PerformerStudio `json:"studios"`
}
func (Performer) IsEditTarget() {}
@@ -359,16 +374,16 @@ type PerformerEditInput struct {
}
type PerformerEditOptions struct {
// Set performer alias on scenes without alias to old name if name is changed
// Set performer alias on scenes without alias to old name if name is changed
SetModifyAliases bool `json:"set_modify_aliases"`
// Set performer alias on scenes attached to merge sources to old name
// Set performer alias on scenes attached to merge sources to old name
SetMergeAliases bool `json:"set_merge_aliases"`
}
type PerformerEditOptionsInput struct {
// Set performer alias on scenes without alias to old name if name is changed
// Set performer alias on scenes without alias to old name if name is changed
SetModifyAliases *bool `json:"set_modify_aliases"`
// Set performer alias on scenes attached to merge sources to old name
// Set performer alias on scenes attached to merge sources to old name
SetMergeAliases *bool `json:"set_merge_aliases"`
}
@@ -402,6 +417,11 @@ type PerformerFilterType struct {
Piercings *BodyModificationCriterionInput `json:"piercings"`
}
type PerformerStudio struct {
Studio *Studio `json:"studio"`
SceneCount int `json:"scene_count"`
}
type PerformerUpdateInput struct {
ID string `json:"id"`
Name *string `json:"name"`
@@ -507,7 +527,7 @@ type SceneCreateInput struct {
Performers []*PerformerAppearanceInput `json:"performers"`
TagIds []string `json:"tag_ids"`
ImageIds []string `json:"image_ids"`
Fingerprints []*FingerprintInput `json:"fingerprints"`
Fingerprints []*FingerprintEditInput `json:"fingerprints"`
Duration *int `json:"duration"`
Director *string `json:"director"`
}
@@ -547,7 +567,7 @@ type SceneEditDetailsInput struct {
Performers []*PerformerAppearanceInput `json:"performers"`
TagIds []string `json:"tag_ids"`
ImageIds []string `json:"image_ids"`
Fingerprints []*FingerprintInput `json:"fingerprints"`
Fingerprints []*FingerprintEditInput `json:"fingerprints"`
Duration *int `json:"duration"`
Director *string `json:"director"`
}
@@ -578,6 +598,8 @@ type SceneFilterType struct {
Performers *MultiIDCriterionInput `json:"performers"`
// Filter to include scenes with performer appearing as alias
Alias *StringCriterionInput `json:"alias"`
// Filter to only include scenes with these fingerprints
Fingerprints *MultiIDCriterionInput `json:"fingerprints"`
}
type SceneUpdateInput struct {
@@ -590,7 +612,7 @@ type SceneUpdateInput struct {
Performers []*PerformerAppearanceInput `json:"performers"`
TagIds []string `json:"tag_ids"`
ImageIds []string `json:"image_ids"`
Fingerprints []*FingerprintInput `json:"fingerprints"`
Fingerprints []*FingerprintEditInput `json:"fingerprints"`
Duration *int `json:"duration"`
Director *string `json:"director"`
}

View File

@@ -75,7 +75,7 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models
return nil, err
}
var fingerprints []string
var fingerprints []*graphql.FingerprintQueryInput
// map fingerprints to their scene index
fpToScene := make(map[string][]int)
@@ -93,18 +93,27 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([][]*models
}
if scene.Checksum.Valid {
fingerprints = append(fingerprints, scene.Checksum.String)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.Checksum.String,
Algorithm: graphql.FingerprintAlgorithmMd5,
})
fpToScene[scene.Checksum.String] = append(fpToScene[scene.Checksum.String], index)
}
if scene.OSHash.Valid {
fingerprints = append(fingerprints, scene.OSHash.String)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.OSHash.String,
Algorithm: graphql.FingerprintAlgorithmOshash,
})
fpToScene[scene.OSHash.String] = append(fpToScene[scene.OSHash.String], index)
}
if scene.Phash.Valid {
phashStr := utils.PhashToString(scene.Phash.Int64)
fingerprints = append(fingerprints, phashStr)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: phashStr,
Algorithm: graphql.FingerprintAlgorithmPhash,
})
fpToScene[phashStr] = append(fpToScene[phashStr], index)
}
}
@@ -147,7 +156,7 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*mode
return nil, err
}
var fingerprints []string
var fingerprints []*graphql.FingerprintQueryInput
if err := c.txnManager.WithReadTxn(ctx, func(r models.ReaderRepository) error {
qb := r.Scene()
@@ -163,16 +172,24 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*mode
}
if scene.Checksum.Valid {
fingerprints = append(fingerprints, scene.Checksum.String)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.Checksum.String,
Algorithm: graphql.FingerprintAlgorithmMd5,
})
}
if scene.OSHash.Valid {
fingerprints = append(fingerprints, scene.OSHash.String)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: scene.OSHash.String,
Algorithm: graphql.FingerprintAlgorithmOshash,
})
}
if scene.Phash.Valid {
phashStr := utils.PhashToString(scene.Phash.Int64)
fingerprints = append(fingerprints, phashStr)
fingerprints = append(fingerprints, &graphql.FingerprintQueryInput{
Hash: utils.PhashToString(scene.Phash.Int64),
Algorithm: graphql.FingerprintAlgorithmPhash,
})
}
}
@@ -184,20 +201,20 @@ func (c Client) FindStashBoxScenesByFingerprintsFlat(sceneIDs []string) ([]*mode
return c.findStashBoxScenesByFingerprints(ctx, fingerprints)
}
func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprints []string) ([]*models.ScrapedScene, error) {
func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, fingerprints []*graphql.FingerprintQueryInput) ([]*models.ScrapedScene, error) {
var ret []*models.ScrapedScene
for i := 0; i < len(fingerprints); i += 100 {
end := i + 100
if end > len(fingerprints) {
end = len(fingerprints)
}
scenes, err := c.client.FindScenesByFingerprints(ctx, fingerprints[i:end])
scenes, err := c.client.FindScenesByFullFingerprints(ctx, fingerprints[i:end])
if err != nil {
return nil, err
}
sceneFragments := scenes.FindScenesByFingerprints
sceneFragments := scenes.FindScenesByFullFingerprints
for _, s := range sceneFragments {
ss, err := sceneFragmentToScrapedScene(ctx, c.getHTTPClient(), c.txnManager, s)