Return scrape results if only relationships are returned (#3954)

* Handle scene scrape results where basic fields unset
* Apply fix to other types
* Show scrape dialog if only new items scraped

pkg/scraper/mapped.go

@@ -774,7 +774,7 @@ func (r mappedResults) setKey(index int, key string, value string) mappedResults
 }
 
 func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*models.ScrapedPerformer, error) {
-	var ret *models.ScrapedPerformer
+	var ret models.ScrapedPerformer
 
 	performerMap := s.Performer
 	if performerMap == nil {
@@ -784,9 +784,6 @@ func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*mod
 	performerTagsMap := performerMap.Tags
 
 	results := performerMap.process(ctx, q, s.Common)
-	if len(results) > 0 {
-		ret = &models.ScrapedPerformer{}
-		results[0].apply(ret)
 
 	// now apply the tags
 	if performerTagsMap != nil {
@@ -799,9 +796,16 @@ func (s mappedScraper) scrapePerformer(ctx context.Context, q mappedQuery) (*mod
 			ret.Tags = append(ret.Tags, tag)
 		}
 	}
+
+	if len(results) == 0 && len(ret.Tags) == 0 {
+		return nil, nil
 	}
 
-	return ret, nil
+	if len(results) > 0 {
+		results[0].apply(&ret)
+	}
+
+	return &ret, nil
 }
 
 func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*models.ScrapedPerformer, error) {
@@ -822,9 +826,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
 	return ret, nil
 }
 
-func (s mappedScraper) processScene(ctx context.Context, q mappedQuery, r mappedResult, resultIndex int) *ScrapedScene {
+// processSceneRelationships sets the relationships on the ScrapedScene. It returns true if any relationships were set.
-	var ret ScrapedScene
+func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *ScrapedScene) bool {
-
 	sceneScraperConfig := s.Scene
 
 	scenePerformersMap := sceneScraperConfig.Performers
@@ -832,44 +835,12 @@ func (s mappedScraper) processScene(ctx context.Context, q mappedQuery, r mapped
 	sceneStudioMap := sceneScraperConfig.Studio
 	sceneMoviesMap := sceneScraperConfig.Movies
 
-	scenePerformerTagsMap := scenePerformersMap.Tags
+	ret.Performers = s.processPerformers(ctx, scenePerformersMap, q)
-
-	r.apply(&ret)
-
-	// process performer tags once
-	var performerTagResults mappedResults
-	if scenePerformerTagsMap != nil {
-		performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common)
-	}
-
-	// now apply the performers and tags
-	if scenePerformersMap.mappedConfig != nil {
-		logger.Debug(`Processing scene performers:`)
-		performerResults := scenePerformersMap.process(ctx, q, s.Common)
-
-		for _, p := range performerResults {
-			performer := &models.ScrapedPerformer{}
-			p.apply(performer)
-
-			for _, p := range performerTagResults {
-				tag := &models.ScrapedTag{}
-				p.apply(tag)
-				performer.Tags = append(performer.Tags, tag)
-			}
-
-			ret.Performers = append(ret.Performers, performer)
-		}
-	}
 
 	if sceneTagsMap != nil {
 		logger.Debug(`Processing scene tags:`)
-		tagResults := sceneTagsMap.process(ctx, q, s.Common)
 
-		for _, p := range tagResults {
+		ret.Tags = processRelationships[models.ScrapedTag](ctx, s, sceneTagsMap, q)
-			tag := &models.ScrapedTag{}
-			p.apply(tag)
-			ret.Tags = append(ret.Tags, tag)
-		}
 	}
 
 	if sceneStudioMap != nil {
@@ -886,16 +857,57 @@ func (s mappedScraper) processScene(ctx context.Context, q mappedQuery, r mapped
 
 	if sceneMoviesMap != nil {
 		logger.Debug(`Processing scene movies:`)
-		movieResults := sceneMoviesMap.process(ctx, q, s.Common)
+		ret.Movies = processRelationships[models.ScrapedMovie](ctx, s, sceneMoviesMap, q)
+	}
 
-		for _, p := range movieResults {
+	return len(ret.Performers) > 0 || len(ret.Tags) > 0 || ret.Studio != nil || len(ret.Movies) > 0
-			movie := &models.ScrapedMovie{}
+}
-			p.apply(movie)
+
-			ret.Movies = append(ret.Movies, movie)
+func (s mappedScraper) processPerformers(ctx context.Context, performersMap mappedPerformerScraperConfig, q mappedQuery) []*models.ScrapedPerformer {
+	var ret []*models.ScrapedPerformer
+
+	// now apply the performers and tags
+	if performersMap.mappedConfig != nil {
+		logger.Debug(`Processing performers:`)
+		performerResults := performersMap.process(ctx, q, s.Common)
+
+		scenePerformerTagsMap := performersMap.Tags
+
+		// process performer tags once
+		var performerTagResults mappedResults
+		if scenePerformerTagsMap != nil {
+			performerTagResults = scenePerformerTagsMap.process(ctx, q, s.Common)
+		}
+
+		for _, p := range performerResults {
+			performer := &models.ScrapedPerformer{}
+			p.apply(performer)
+
+			for _, p := range performerTagResults {
+				tag := &models.ScrapedTag{}
+				p.apply(tag)
+				performer.Tags = append(performer.Tags, tag)
+			}
+
+			ret = append(ret, performer)
 		}
 	}
 
-	return &ret
+	return ret
+}
+
+func processRelationships[T any](ctx context.Context, s mappedScraper, relationshipMap mappedConfig, q mappedQuery) []*T {
+	var ret []*T
+
+	results := relationshipMap.process(ctx, q, s.Common)
+
+	for _, p := range results {
+		var value T
+		p.apply(&value)
+		ret = append(ret, &value)
+	}
+
+	return ret
 }
 
 func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*ScrapedScene, error) {
@@ -911,15 +923,17 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
 	results := sceneMap.process(ctx, q, s.Common)
 	for i, r := range results {
 		logger.Debug(`Processing scene:`)
-		ret = append(ret, s.processScene(ctx, q, r, i))
+
+		var thisScene ScrapedScene
+		r.apply(&thisScene)
+		s.processSceneRelationships(ctx, q, i, &thisScene)
+		ret = append(ret, &thisScene)
 	}
 
 	return ret, nil
 }
 
 func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*ScrapedScene, error) {
-	var ret *ScrapedScene
-
 	sceneScraperConfig := s.Scene
 	sceneMap := sceneScraperConfig.mappedConfig
 	if sceneMap == nil {
@@ -928,15 +942,25 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
 
 	logger.Debug(`Processing scene:`)
 	results := sceneMap.process(ctx, q, s.Common)
+
+	var ret ScrapedScene
 	if len(results) > 0 {
-		ret = s.processScene(ctx, q, results[0], 0)
+		results[0].apply(&ret)
+	}
+	hasRelationships := s.processSceneRelationships(ctx, q, 0, &ret)
+
+	// #3953 - process only returns results if the non-relationship fields are
+	// populated
+	// only return if we have results or relationships
+	if len(results) > 0 || hasRelationships {
+		return &ret, nil
 	}
 
-	return ret, nil
+	return nil, nil
 }
 
 func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
-	var ret *ScrapedGallery
+	var ret ScrapedGallery
 
 	galleryScraperConfig := s.Gallery
 	galleryMap := galleryScraperConfig.mappedConfig
@@ -950,10 +974,6 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*Scrap
 
 	logger.Debug(`Processing gallery:`)
 	results := galleryMap.process(ctx, q, s.Common)
-	if len(results) > 0 {
-		ret = &ScrapedGallery{}
-
-		results[0].apply(ret)
 
 	// now apply the performers and tags
 	if galleryPerformersMap != nil {
@@ -988,13 +1008,21 @@ func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*Scrap
 			ret.Studio = studio
 		}
 	}
+
+	// if no basic fields are populated, and no relationships, then return nil
+	if len(results) == 0 && len(ret.Performers) == 0 && len(ret.Tags) == 0 && ret.Studio == nil {
+		return nil, nil
 	}
 
-	return ret, nil
+	if len(results) > 0 {
+		results[0].apply(&ret)
+	}
+
+	return &ret, nil
 }
 
 func (s mappedScraper) scrapeMovie(ctx context.Context, q mappedQuery) (*models.ScrapedMovie, error) {
-	var ret *models.ScrapedMovie
+	var ret models.ScrapedMovie
 
 	movieScraperConfig := s.Movie
 	movieMap := movieScraperConfig.mappedConfig
@@ -1005,9 +1033,6 @@ func (s mappedScraper) scrapeMovie(ctx context.Context, q mappedQuery) (*models.
 	movieStudioMap := movieScraperConfig.Studio
 
 	results := movieMap.process(ctx, q, s.Common)
-	if len(results) > 0 {
-		ret = &models.ScrapedMovie{}
-		results[0].apply(ret)
 
 	if movieStudioMap != nil {
 		logger.Debug(`Processing movie studio:`)
@@ -1019,7 +1044,14 @@ func (s mappedScraper) scrapeMovie(ctx context.Context, q mappedQuery) (*models.
 			ret.Studio = studio
 		}
 	}
+
+	if len(results) == 0 && ret.Studio == nil {
+		return nil, nil
 	}
 
-	return ret, nil
+	if len(results) > 0 {
+		results[0].apply(&ret)
+	}
+
+	return &ret, nil
 }

ui/v2.5/src/components/Galleries/GalleryDetails/GalleryScrapeDialog.tsx

@@ -281,7 +281,10 @@ export const GalleryScrapeDialog: React.FC<IGalleryScrapeDialogProps> = (
   if (
     [title, url, date, studio, performers, tags, details].every(
       (r) => !r.scraped
-    )
+    ) &&
+    !newStudio &&
+    newPerformers.length === 0 &&
+    newTags.length === 0
   ) {
     props.onClose();
     return <></>;

ui/v2.5/src/components/Movies/MovieDetails/MovieScrapeDialog.tsx

@@ -162,7 +162,7 @@ export const MovieScrapeDialog: React.FC<IMovieScrapeDialogProps> = (
     backImage,
   ];
   // don't show the dialog if nothing was scraped
-  if (allFields.every((r) => !r.scraped)) {
+  if (allFields.every((r) => !r.scraped) && !newStudio) {
     props.onClose();
     return <></>;
   }

ui/v2.5/src/components/Performers/PerformerDetails/PerformerScrapeDialog.tsx

@@ -443,7 +443,7 @@ export const PerformerScrapeDialog: React.FC<IPerformerScrapeDialogProps> = (
     remoteSiteID,
   ];
   // don't show the dialog if nothing was scraped
-  if (allFields.every((r) => !r.scraped)) {
+  if (allFields.every((r) => !r.scraped) && newTags.length === 0) {
     props.onClose();
     return <></>;
   }

ui/v2.5/src/components/Scenes/SceneDetails/SceneScrapeDialog.tsx

@@ -426,7 +426,11 @@ export const SceneScrapeDialog: React.FC<ISceneScrapeDialogProps> = ({
       details,
       image,
       stashID,
-    ].every((r) => !r.scraped)
+    ].every((r) => !r.scraped) &&
+    newTags.length === 0 &&
+    newPerformers.length === 0 &&
+    newMovies.length === 0 &&
+    !newStudio
   ) {
     onClose();
     return <></>;