Fix outstanding tagger issues (#912)

* Fix potential image errors
* Fix issue preventing favoriting of tagged performers
* Add error handling in case of network issues
* Show individual search errors
* Unset scene results if query fails
* Don't abort scene submission if scene id isn't found

pkg/models/querybuilder_performer.go

@@ -175,6 +175,7 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin
 	query.body += `
 		left join performers_scenes as scenes_join on scenes_join.performer_id = performers.id
 		left join scenes on scenes_join.scene_id = scenes.id
+		left join performer_stash_ids on performer_stash_ids.performer_id = performers.id
 	`
 
 	if q := findFilter.Q; q != nil && *q != "" {
@@ -219,15 +220,14 @@ func (qb *PerformerQueryBuilder) Query(performerFilter *PerformerFilterType, fin
 			query.body += `left join performers_image on performers_image.performer_id = performers.id
 			`
 			query.addWhere("performers_image.performer_id IS NULL")
+		case "stash_id":
+			query.addWhere("performer_stash_ids.performer_id IS NULL")
 		default:
 			query.addWhere("performers." + *isMissingFilter + " IS NULL OR TRIM(performers." + *isMissingFilter + ") = ''")
 		}
 	}
 
 	if stashIDFilter := performerFilter.StashID; stashIDFilter != nil {
-		query.body += `
-			JOIN performer_stash_ids on performer_stash_ids.performer_id = performers.id
-		`
 		query.addWhere("performer_stash_ids.stash_id = ?")
 		query.addArg(stashIDFilter)
 	}

pkg/models/querybuilder_scene.go

@@ -298,6 +298,7 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin
 		left join studios as studio on studio.id = scenes.studio_id
 		left join galleries as gallery on gallery.scene_id = scenes.id
 		left join scenes_tags as tags_join on tags_join.scene_id = scenes.id
+		left join scene_stash_ids on scene_stash_ids.scene_id = scenes.id
 	`
 
 	if q := findFilter.Q; q != nil && *q != "" {
@@ -356,6 +357,8 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin
 			query.addWhere("scenes.date IS \"\" OR scenes.date IS \"0001-01-01\"")
 		case "tags":
 			query.addWhere("tags_join.scene_id IS NULL")
+		case "stash_id":
+			query.addWhere("scene_stash_ids.scene_id IS NULL")
 		default:
 			query.addWhere("scenes." + *isMissingFilter + " IS NULL OR TRIM(scenes." + *isMissingFilter + ") = ''")
 		}
@@ -405,9 +408,6 @@ func (qb *SceneQueryBuilder) Query(sceneFilter *SceneFilterType, findFilter *Fin
 	}
 
 	if stashIDFilter := sceneFilter.StashID; stashIDFilter != nil {
-		query.body += `
-			JOIN scene_stash_ids on scene_stash_ids.scene_id = scenes.id
-		`
 		query.addWhere("scene_stash_ids.stash_id = ?")
 		query.addArg(stashIDFilter)
 	}

pkg/models/querybuilder_studio.go

@@ -158,6 +158,7 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter *
 	body := selectDistinctIDs("studios")
 	body += `
 		left join scenes on studios.id = scenes.studio_id		
+		left join studio_stash_ids on studio_stash_ids.studio_id = studios.id
 	`
 
 	if q := findFilter.Q; q != nil && *q != "" {
@@ -183,9 +184,6 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter *
 	}
 
 	if stashIDFilter := studioFilter.StashID; stashIDFilter != nil {
-		body += `
-			JOIN studio_stash_ids on studio_stash_ids.studio_id = studios.id
-		`
 		whereClauses = append(whereClauses, "studio_stash_ids.stash_id = ?")
 		args = append(args, stashIDFilter)
 	}
@@ -196,6 +194,8 @@ func (qb *StudioQueryBuilder) Query(studioFilter *StudioFilterType, findFilter *
 			body += `left join studios_image on studios_image.studio_id = studios.id
 			`
 			whereClauses = appendClause(whereClauses, "studios_image.studio_id IS NULL")
+		case "stash_id":
+			whereClauses = appendClause(whereClauses, "studio_stash_ids.studio_id IS NULL")
 		default:
 			whereClauses = appendClause(whereClauses, "studios."+*isMissingFilter+" IS NULL")
 		}

pkg/scraper/stashbox/stash_box.go

@@ -93,21 +93,27 @@ func (c Client) FindStashBoxScenesByFingerprints(sceneIDs []string) ([]*models.S
 }
 
 func (c Client) findStashBoxScenesByFingerprints(fingerprints []string) ([]*models.ScrapedScene, error) {
-	scenes, err := c.client.FindScenesByFingerprints(context.TODO(), fingerprints)
-
-	if err != nil {
-		return nil, err
-	}
-
-	sceneFragments := scenes.FindScenesByFingerprints
-
 	var ret []*models.ScrapedScene
-	for _, s := range sceneFragments {
+	for i := 0; i < len(fingerprints); i += 100 {
-		ss, err := sceneFragmentToScrapedScene(s)
+		end := i + 100
+		if end > len(fingerprints) {
+			end = len(fingerprints)
+		}
+		scenes, err := c.client.FindScenesByFingerprints(context.TODO(), fingerprints[i:end])
+
 		if err != nil {
 			return nil, err
 		}
-		ret = append(ret, ss)
+
+		sceneFragments := scenes.FindScenesByFingerprints
+
+		for _, s := range sceneFragments {
+			ss, err := sceneFragmentToScrapedScene(s)
+			if err != nil {
+				return nil, err
+			}
+			ret = append(ret, ss)
+		}
 	}
 
 	return ret, nil
@@ -127,7 +133,7 @@ func (c Client) SubmitStashBoxFingerprints(sceneIDs []string, endpoint string) (
 		}
 
 		if scene == nil {
-			return false, fmt.Errorf("scene with id %d not found", idInt)
+			continue
 		}
 
 		stashIDs, err := jqb.GetSceneStashIDs(idInt)

ui/v2.5/src/components/Performers/PerformerDetails/Performer.tsx

@@ -106,7 +106,13 @@ export const Performer: React.FC = () => {
     try {
       if (!isNew) {
         await updatePerformer({
-          variables: performerInput as GQL.PerformerUpdateInput,
+          variables: {
+            ...performerInput,
+            stash_ids: (performerInput?.stash_ids ?? []).map((s) => ({
+              endpoint: s.endpoint,
+              stash_id: s.stash_id,
+            })),
+          } as GQL.PerformerUpdateInput,
         });
         if (performerInput.image) {
           // Refetch image to bust browser cache

ui/v2.5/src/components/Tagger/StashSearchResult.tsx

@@ -1,4 +1,4 @@
-import React, { useCallback, useState } from "react";
+import React, { useState, useReducer } from "react";
 import cx from "classnames";
 import { Button } from "react-bootstrap";
 import { uniq } from "lodash";
@@ -64,6 +64,16 @@ interface IStashSearchResultProps {
   queueFingerprintSubmission: (sceneId: string, endpoint: string) => void;
 }
 
+interface IPerformerReducerAction {
+  id: string;
+  data: PerformerOperation;
+}
+
+const performerReducer = (
+  state: Record<string, PerformerOperation>,
+  action: IPerformerReducerAction
+) => ({ ...state, [action.id]: action.data });
+
 const StashSearchResult: React.FC<IStashSearchResultProps> = ({
   scene,
   stashScene,
@@ -78,9 +88,7 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
   queueFingerprintSubmission,
 }) => {
   const [studio, setStudio] = useState<StudioOperation>();
-  const [performers, setPerformers] = useState<
+  const [performers, dispatch] = useReducer(performerReducer, {});
-    Record<string, PerformerOperation>
-  >({});
   const [saveState, setSaveState] = useState<string>("");
   const [error, setError] = useState<{ message?: string; details?: string }>(
     {}
@@ -96,11 +104,10 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
   });
   const { data: allTags } = GQL.useAllTagsForFilterQuery();
 
-  const setPerformer = useCallback(
+  const setPerformer = (
-    (performerData: PerformerOperation, performerID: string) =>
+    performerData: PerformerOperation,
-      setPerformers({ ...performers, [performerID]: performerData }),
+    performerID: string
-    [performers]
+  ) => dispatch({ id: performerID, data: performerData });
-  );
 
   const handleSave = async () => {
     setError({});
@@ -232,7 +239,8 @@ const StashSearchResult: React.FC<IStashSearchResultProps> = ({
         });
         if (img.status === 200) {
           const blob = await img.blob();
-          imgData = await blobToBase64(blob);
+          // Sanity check on image size since bad images will fail
+          if (blob.size > 10000) imgData = await blobToBase64(blob);
         }
       }
 

ui/v2.5/src/components/Tagger/Tagger.tsx

@@ -76,12 +76,16 @@ const TaggerList: React.FC<ITaggerListProps> = ({
   queueFingerprintSubmission,
   clearSubmissionQueue,
 }) => {
+  const [fingerprintError, setFingerprintError] = useState("");
   const [loading, setLoading] = useState(false);
   const [queryString, setQueryString] = useState<Record<string, string>>({});
 
   const [searchResults, setSearchResults] = useState<
     Record<string, IStashBoxScene[]>
   >({});
+  const [searchErrors, setSearchErrors] = useState<
+    Record<string, string | undefined>
+  >({});
   const [selectedResult, setSelectedResult] = useState<
     Record<string, number>
   >();
@@ -96,14 +100,29 @@ const TaggerList: React.FC<ITaggerListProps> = ({
     config.fingerprintQueue[selectedEndpoint.endpoint] ?? [];
 
   const doBoxSearch = (sceneID: string, searchVal: string) => {
-    stashBoxQuery(searchVal, selectedEndpoint.index).then((queryData) => {
+    stashBoxQuery(searchVal, selectedEndpoint.index)
-      const s = selectScenes(queryData.data?.queryStashBoxScene);
+      .then((queryData) => {
-      setSearchResults({
+        const s = selectScenes(queryData.data?.queryStashBoxScene);
-        ...searchResults,
+        setSearchResults({
-        [sceneID]: s,
+          ...searchResults,
+          [sceneID]: s,
+        });
+        setSearchErrors({
+          ...searchErrors,
+          [sceneID]: undefined,
+        });
+        setLoading(false);
+      })
+      .catch(() => {
+        setLoading(false);
+        // Destructure to remove existing result
+        const { [sceneID]: unassign, ...results } = searchResults;
+        setSearchResults(results);
+        setSearchErrors({
+          ...searchErrors,
+          [sceneID]: "Network Error",
+        });
       });
-      setLoading(false);
-    });
 
     setLoading(true);
   };
@@ -113,9 +132,13 @@ const TaggerList: React.FC<ITaggerListProps> = ({
     { loading: submittingFingerprints },
   ] = GQL.useSubmitStashBoxFingerprintsMutation({
     onCompleted: (result) => {
+      setFingerprintError("");
       if (result.submitStashBoxFingerprints)
         clearSubmissionQueue(selectedEndpoint.endpoint);
     },
+    onError: () => {
+      setFingerprintError("Network Error");
+    },
   });
 
   const handleFingerprintSubmission = () => {
@@ -141,20 +164,36 @@ const TaggerList: React.FC<ITaggerListProps> = ({
     const newFingerprints = { ...fingerprints };
 
     const sceneIDs = scenes
-      .filter(
+      .filter((s) => s.stash_ids.length === 0)
-        (s) => fingerprints[s.id] === undefined && s.stash_ids.length === 0
-      )
       .map((s) => s.id);
 
-    const results = await stashBoxBatchQuery(sceneIDs, selectedEndpoint.index);
+    const results = await stashBoxBatchQuery(
+      sceneIDs,
+      selectedEndpoint.index
+    ).catch(() => {
+      setLoadingFingerprints(false);
+      setFingerprintError("Network Error");
+    });
+
+    if (!results) return;
+
+    // clear search errors
+    setSearchErrors({});
+
     selectScenes(results.data?.queryStashBoxScene).forEach((scene) => {
       scene.fingerprints?.forEach((f) => {
         newFingerprints[f.hash] = scene;
       });
     });
 
+    // Null any ids that are still undefined since it means they weren't found
+    sceneIDs.forEach((id) => {
+      newFingerprints[id] = newFingerprints[id] ?? null;
+    });
+
     setFingerprints(newFingerprints);
     setLoadingFingerprints(false);
+    setFingerprintError("");
   };
 
   const canFingerprintSearch = () =>
@@ -164,7 +203,10 @@ const TaggerList: React.FC<ITaggerListProps> = ({
 
   const getFingerprintCount = () => {
     const count = scenes.filter(
-      (s) => s.stash_ids.length === 0 && fingerprints[s.id]
+      (s) =>
+        s.stash_ids.length === 0 &&
+        ((s.checksum && fingerprints[s.checksum]) ||
+          (s.oshash && fingerprints[s.oshash]))
     ).length;
     return `${count > 0 ? count : "No"} new fingerprint matches found`;
   };
@@ -246,11 +288,13 @@ const TaggerList: React.FC<ITaggerListProps> = ({
       }
 
       let searchResult;
-      if (searchResults[scene.id]?.length === 0)
+      if (searchErrors[scene.id]) {
         searchResult = (
-          <div className="text-danger font-weight-bold">No results found.</div>
+          <div className="text-danger font-weight-bold">
+            {searchErrors[scene.id]}
+          </div>
         );
-      else if (fingerprintMatch && !isTagged && !hasStashIDs) {
+      } else if (fingerprintMatch && !isTagged && !hasStashIDs) {
         searchResult = (
           <StashSearchResult
             showMales={config.showMales}
@@ -266,10 +310,17 @@ const TaggerList: React.FC<ITaggerListProps> = ({
             queueFingerprintSubmission={queueFingerprintSubmission}
           />
         );
-      } else if (searchResults[scene.id] && !isTagged && !fingerprintMatch) {
+      } else if (
+        searchResults[scene.id]?.length > 0 &&
+        !isTagged &&
+        !fingerprintMatch
+      ) {
         searchResult = (
           <ul className="pl-0 mt-4">
-            {sortScenesByDuration(searchResults[scene.id]).map(
+            {sortScenesByDuration(
+              searchResults[scene.id],
+              scene.file.duration ?? undefined
+            ).map(
               (sceneResult, i) =>
                 sceneResult && (
                   <StashSearchResult
@@ -295,6 +346,10 @@ const TaggerList: React.FC<ITaggerListProps> = ({
             )}
           </ul>
         );
+      } else if (searchResults[scene.id]?.length === 0) {
+        searchResult = (
+          <div className="text-danger font-weight-bold">No results found.</div>
+        );
       }
 
       return (
@@ -320,14 +375,15 @@ const TaggerList: React.FC<ITaggerListProps> = ({
 
   return (
     <Card className="tagger-table">
-      <div className="tagger-table-header row mb-4">
+      <div className="tagger-table-header row flex-nowrap mb-4 align-items-center">
         <div className="col-md-6">
           <b>Path</b>
         </div>
         <div className="col-md-2">
           <b>Query</b>
         </div>
-        <div className="ml-auto mr-2">
+        <b className="ml-auto mr-2 text-danger">{fingerprintError}</b>
+        <div className="mr-2">
           {fingerprintQueue.length > 0 && (
             <Button
               onClick={handleFingerprintSubmission}

ui/v2.5/src/components/Tagger/utils.ts

@@ -159,9 +159,9 @@ export const sortScenesByDuration = (
 ) =>
   scenes.sort((a, b) => {
     const adur =
-      a?.duration ?? a?.fingerprints.map((f) => f.duration)?.[0] ?? null;
+      a?.duration || (a?.fingerprints.map((f) => f.duration)?.[0] ?? null);
     const bdur =
-      b?.duration ?? b?.fingerprints.map((f) => f.duration)?.[0] ?? null;
+      b?.duration || (b?.fingerprints.map((f) => f.duration)?.[0] ?? null);
     if (!adur && !bdur) return 0;
     if (adur && !bdur) return -1;
     if (!adur && bdur) return 1;

ui/v2.5/src/models/list-filter/criteria/is-missing.ts

@@ -20,6 +20,7 @@ export class SceneIsMissingCriterion extends IsMissingCriterion {
     "movie",
     "performers",
     "tags",
+    "stash_id",
   ];
 }
 
@@ -103,7 +104,7 @@ export class TagIsMissingCriterionOption implements ICriterionOption {
 
 export class StudioIsMissingCriterion extends IsMissingCriterion {
   public type: CriterionType = "studioIsMissing";
-  public options: string[] = ["image"];
+  public options: string[] = ["image", "stash_id"];
 }
 
 export class StudioIsMissingCriterionOption implements ICriterionOption {