From 9ec762ae9ac7d04ab56f133e97861a99a4c7100d Mon Sep 17 00:00:00 2001 From: InfiniteTF Date: Wed, 4 Nov 2020 22:28:58 +0100 Subject: [PATCH] Fix outstanding tagger issues (#912) * Fix potential image errors * Fix issue preventing favoriting of tagged performers * Add error handling in case of network issues * Show individual search errors * Unset scene results if query fails * Don't abort scene submission if scene id isn't found --- pkg/models/querybuilder_performer.go | 6 +- pkg/models/querybuilder_scene.go | 6 +- pkg/models/querybuilder_studio.go | 6 +- pkg/scraper/stashbox/stash_box.go | 30 +++--- .../Performers/PerformerDetails/Performer.tsx | 8 +- .../components/Tagger/StashSearchResult.tsx | 28 ++++-- ui/v2.5/src/components/Tagger/Tagger.tsx | 94 +++++++++++++++---- ui/v2.5/src/components/Tagger/utils.ts | 4 +- .../models/list-filter/criteria/is-missing.ts | 3 +- 9 files changed, 131 insertions(+), 54 deletions(-) diff --git a/pkg/models/querybuilder_performer.go b/pkg/models/querybuilder_performer.go index 8729fd5da..7bebf214b 100644 --- a/pkg/models/querybuilder_performer.go +++ b/pkg/models/querybuilder_performer.go @@ -175,6 +175,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin query.body += ` left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id left join scenes on scenes_join.scene_id = scenes.id + left join performer_stash_ids on performer_stash_ids.performer_id = performers.id ` if q := findFilter.Q; q != nil && *q != "" { @@ -219,15 +220,14 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin query.body += `left join performers_image on performers_image.performer_id = performers.id ` query.addWhere("performers_image.performer_id IS NULL") + case "stash_id": + query.addWhere("performer_stash_ids.performer_id IS NULL") default: query.addWhere("performers." + *isMissingFilter + " IS NULL OR TRIM(performers." + *isMissingFilter + ") = ''") } } if stashIDFilter := performerFilter.StashID; stashIDFilter != nil { - query.body += ` - JOIN performer_stash_ids on performer_stash_ids.performer_id = performers.id - ` query.addWhere("performer_stash_ids.stash_id = ?") query.addArg(stashIDFilter) } diff --git a/pkg/models/querybuilder_scene.go b/pkg/models/querybuilder_scene.go index ad336f816..a44b6b2c1 100644 --- a/pkg/models/querybuilder_scene.go +++ b/pkg/models/querybuilder_scene.go @@ -298,6 +298,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin left join studios as studio on studio.id = scenes.studio_id left join galleries as gallery on gallery.scene_id = scenes.id left join scenes_tags as tags_join on tags_join.scene_id = scenes.id + left join scene_stash_ids on scene_stash_ids.scene_id = scenes.id ` if q := findFilter.Q; q != nil && *q != "" { @@ -356,6 +357,8 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin query.addWhere("scenes.date IS \"\" OR scenes.date IS \"0001-01-01\"") case "tags": query.addWhere("tags_join.scene_id IS NULL") + case "stash_id": + query.addWhere("scene_stash_ids.scene_id IS NULL") default: query.addWhere("scenes." + *isMissingFilter + " IS NULL OR TRIM(scenes." + *isMissingFilter + ") = ''") } @@ -405,9 +408,6 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin } if stashIDFilter := sceneFilter.StashID; stashIDFilter != nil { - query.body += ` - JOIN scene_stash_ids on scene_stash_ids.scene_id = scenes.id - ` query.addWhere("scene_stash_ids.stash_id = ?") query.addArg(stashIDFilter) } diff --git a/pkg/models/querybuilder_studio.go b/pkg/models/querybuilder_studio.go index e8b36e9b4..39528811a 100644 --- a/pkg/models/querybuilder_studio.go +++ b/pkg/models/querybuilder_studio.go @@ -158,6 +158,7 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter * body := selectDistinctIDs("studios") body += ` left join scenes on studios.id = scenes.studio_id + left join studio_stash_ids on studio_stash_ids.studio_id = studios.id ` if q := findFilter.Q; q != nil && *q != "" { @@ -183,9 +184,6 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter * } if stashIDFilter := studioFilter.StashID; stashIDFilter != nil { - body += ` - JOIN studio_stash_ids on studio_stash_ids.studio_id = studios.id - ` whereClauses = append(whereClauses, "studio_stash_ids.stash_id = ?") args = append(args, stashIDFilter) } @@ -196,6 +194,8 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter * body += `left join studios_image on studios_image.studio_id = studios.id ` whereClauses = appendClause(whereClauses, "studios_image.studio_id IS NULL") + case "stash_id": + whereClauses = appendClause(whereClauses, "studio_stash_ids.studio_id IS NULL") default: whereClauses = appendClause(whereClauses, "studios."+*isMissingFilter+" IS NULL") } diff --git a/pkg/scraper/stashbox/stash_box.go b/pkg/scraper/stashbox/stash_box.go index 34c9d8519..4b99d557d 100644 --- a/pkg/scraper/stashbox/stash_box.go +++ b/pkg/scraper/stashbox/stash_box.go @@ -93,21 +93,27 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.S } func (c Client) findStashBoxScenesByFingerprints(fingerprints []string) ([]*models.ScrapedScene, error) { - scenes, err := c.client.FindScenesByFingerprints(context.TODO(), fingerprints) - - if err != nil { - return nil, err - } - - sceneFragments := scenes.FindScenesByFingerprints - var ret []*models.ScrapedScene - for _, s := range sceneFragments { - ss, err := sceneFragmentToScrapedScene(s) + for i := 0; i < len(fingerprints); i += 100 { + end := i + 100 + if end > len(fingerprints) { + end = len(fingerprints) + } + scenes, err := c.client.FindScenesByFingerprints(context.TODO(), fingerprints[i:end]) + if err != nil { return nil, err } - ret = append(ret, ss) + + sceneFragments := scenes.FindScenesByFingerprints + + for _, s := range sceneFragments { + ss, err := sceneFragmentToScrapedScene(s) + if err != nil { + return nil, err + } + ret = append(ret, ss) + } } return ret, nil @@ -127,7 +133,7 @@ func (c Client) SubmitStashBoxFingerprints(sceneIDs []string, endpoint string) ( } if scene == nil { - return false, fmt.Errorf("scene with id %d not found", idInt) + continue } stashIDs, err := jqb.GetSceneStashIDs(idInt) diff --git a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx index dd8e53b00..e810895ce 100644 --- a/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx +++ b/ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx @@ -106,7 +106,13 @@ export const Performer: React.FC = () => { try { if (!isNew) { await updatePerformer({ - variables: performerInput as GQL.PerformerUpdateInput, + variables: { + ...performerInput, + stash_ids: (performerInput?.stash_ids ?? []).map((s) => ({ + endpoint: s.endpoint, + stash_id: s.stash_id, + })), + } as GQL.PerformerUpdateInput, }); if (performerInput.image) { // Refetch image to bust browser cache diff --git a/ui/v2.5/src/components/Tagger/StashSearchResult.tsx b/ui/v2.5/src/components/Tagger/StashSearchResult.tsx index 63bf6940b..1a5717219 100755 --- a/ui/v2.5/src/components/Tagger/StashSearchResult.tsx +++ b/ui/v2.5/src/components/Tagger/StashSearchResult.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useState } from "react"; +import React, { useState, useReducer } from "react"; import cx from "classnames"; import { Button } from "react-bootstrap"; import { uniq } from "lodash"; @@ -64,6 +64,16 @@ interface IStashSearchResultProps { queueFingerprintSubmission: (sceneId: string, endpoint: string) => void; } +interface IPerformerReducerAction { + id: string; + data: PerformerOperation; +} + +const performerReducer = ( + state: Record, + action: IPerformerReducerAction +) => ({ ...state, [action.id]: action.data }); + const StashSearchResult: React.FC = ({ scene, stashScene, @@ -78,9 +88,7 @@ const StashSearchResult: React.FC = ({ queueFingerprintSubmission, }) => { const [studio, setStudio] = useState(); - const [performers, setPerformers] = useState< - Record - >({}); + const [performers, dispatch] = useReducer(performerReducer, {}); const [saveState, setSaveState] = useState(""); const [error, setError] = useState<{ message?: string; details?: string }>( {} @@ -96,11 +104,10 @@ const StashSearchResult: React.FC = ({ }); const { data: allTags } = GQL.useAllTagsForFilterQuery(); - const setPerformer = useCallback( - (performerData: PerformerOperation, performerID: string) => - setPerformers({ ...performers, [performerID]: performerData }), - [performers] - ); + const setPerformer = ( + performerData: PerformerOperation, + performerID: string + ) => dispatch({ id: performerID, data: performerData }); const handleSave = async () => { setError({}); @@ -232,7 +239,8 @@ const StashSearchResult: React.FC = ({ }); if (img.status === 200) { const blob = await img.blob(); - imgData = await blobToBase64(blob); + // Sanity check on image size since bad images will fail + if (blob.size > 10000) imgData = await blobToBase64(blob); } } diff --git a/ui/v2.5/src/components/Tagger/Tagger.tsx b/ui/v2.5/src/components/Tagger/Tagger.tsx index 056bff475..6fb7164ee 100755 --- a/ui/v2.5/src/components/Tagger/Tagger.tsx +++ b/ui/v2.5/src/components/Tagger/Tagger.tsx @@ -76,12 +76,16 @@ const TaggerList: React.FC = ({ queueFingerprintSubmission, clearSubmissionQueue, }) => { + const [fingerprintError, setFingerprintError] = useState(""); const [loading, setLoading] = useState(false); const [queryString, setQueryString] = useState>({}); const [searchResults, setSearchResults] = useState< Record >({}); + const [searchErrors, setSearchErrors] = useState< + Record + >({}); const [selectedResult, setSelectedResult] = useState< Record >(); @@ -96,14 +100,29 @@ const TaggerList: React.FC = ({ config.fingerprintQueue[selectedEndpoint.endpoint] ?? []; const doBoxSearch = (sceneID: string, searchVal: string) => { - stashBoxQuery(searchVal, selectedEndpoint.index).then((queryData) => { - const s = selectScenes(queryData.data?.queryStashBoxScene); - setSearchResults({ - ...searchResults, - [sceneID]: s, + stashBoxQuery(searchVal, selectedEndpoint.index) + .then((queryData) => { + const s = selectScenes(queryData.data?.queryStashBoxScene); + setSearchResults({ + ...searchResults, + [sceneID]: s, + }); + setSearchErrors({ + ...searchErrors, + [sceneID]: undefined, + }); + setLoading(false); + }) + .catch(() => { + setLoading(false); + // Destructure to remove existing result + const { [sceneID]: unassign, ...results } = searchResults; + setSearchResults(results); + setSearchErrors({ + ...searchErrors, + [sceneID]: "Network Error", + }); }); - setLoading(false); - }); setLoading(true); }; @@ -113,9 +132,13 @@ const TaggerList: React.FC = ({ { loading: submittingFingerprints }, ] = GQL.useSubmitStashBoxFingerprintsMutation({ onCompleted: (result) => { + setFingerprintError(""); if (result.submitStashBoxFingerprints) clearSubmissionQueue(selectedEndpoint.endpoint); }, + onError: () => { + setFingerprintError("Network Error"); + }, }); const handleFingerprintSubmission = () => { @@ -141,20 +164,36 @@ const TaggerList: React.FC = ({ const newFingerprints = { ...fingerprints }; const sceneIDs = scenes - .filter( - (s) => fingerprints[s.id] === undefined && s.stash_ids.length === 0 - ) + .filter((s) => s.stash_ids.length === 0) .map((s) => s.id); - const results = await stashBoxBatchQuery(sceneIDs, selectedEndpoint.index); + const results = await stashBoxBatchQuery( + sceneIDs, + selectedEndpoint.index + ).catch(() => { + setLoadingFingerprints(false); + setFingerprintError("Network Error"); + }); + + if (!results) return; + + // clear search errors + setSearchErrors({}); + selectScenes(results.data?.queryStashBoxScene).forEach((scene) => { scene.fingerprints?.forEach((f) => { newFingerprints[f.hash] = scene; }); }); + // Null any ids that are still undefined since it means they weren't found + sceneIDs.forEach((id) => { + newFingerprints[id] = newFingerprints[id] ?? null; + }); + setFingerprints(newFingerprints); setLoadingFingerprints(false); + setFingerprintError(""); }; const canFingerprintSearch = () => @@ -164,7 +203,10 @@ const TaggerList: React.FC = ({ const getFingerprintCount = () => { const count = scenes.filter( - (s) => s.stash_ids.length === 0 && fingerprints[s.id] + (s) => + s.stash_ids.length === 0 && + ((s.checksum && fingerprints[s.checksum]) || + (s.oshash && fingerprints[s.oshash])) ).length; return `${count > 0 ? count : "No"} new fingerprint matches found`; }; @@ -246,11 +288,13 @@ const TaggerList: React.FC = ({ } let searchResult; - if (searchResults[scene.id]?.length === 0) + if (searchErrors[scene.id]) { searchResult = ( -
No results found.
+
+ {searchErrors[scene.id]} +
); - else if (fingerprintMatch && !isTagged && !hasStashIDs) { + } else if (fingerprintMatch && !isTagged && !hasStashIDs) { searchResult = ( = ({ queueFingerprintSubmission={queueFingerprintSubmission} /> ); - } else if (searchResults[scene.id] && !isTagged && !fingerprintMatch) { + } else if ( + searchResults[scene.id]?.length > 0 && + !isTagged && + !fingerprintMatch + ) { searchResult = (
    - {sortScenesByDuration(searchResults[scene.id]).map( + {sortScenesByDuration( + searchResults[scene.id], + scene.file.duration ?? undefined + ).map( (sceneResult, i) => sceneResult && ( = ({ )}
); + } else if (searchResults[scene.id]?.length === 0) { + searchResult = ( +
No results found.
+ ); } return ( @@ -320,14 +375,15 @@ const TaggerList: React.FC = ({ return ( -
+
Path
Query
-
+ {fingerprintError} +
{fingerprintQueue.length > 0 && (