Autotag scraper (#1817)

* Refactor scraper structures * Move matching code into new package * Add autotag scraper * Always check first letter of auto-tag names * Account for nulls Co-authored-by: Kermie <kermie@isinthe.house>
2025-12-17 12:24:38 +03:00 · 2021-10-11 23:06:06 +11:00
parent b5381ff071
commit e9d48683f8
22 changed files with 1023 additions and 660 deletions
--- a/pkg/scraper/config_scraper.go
+++ b/pkg/scraper/config_scraper.go
@@ -0,0 +1,283 @@
+package scraper
+
+import "github.com/stashapp/stash/pkg/models"
+
+type configSceneScraper struct {
+	*configScraper
+}
+
+func (c *configSceneScraper) matchesURL(url string) bool {
+	return c.config.matchesSceneURL(url)
+}
+
+func (c *configSceneScraper) scrapeByName(name string) ([]*models.ScrapedScene, error) {
+	if c.config.SceneByName != nil {
+		s := c.config.getScraper(*c.config.SceneByName, c.txnManager, c.globalConfig)
+		return s.scrapeScenesByName(name)
+	}
+
+	return nil, nil
+}
+
+func (c *configSceneScraper) scrapeByScene(scene *models.Scene) (*models.ScrapedScene, error) {
+	if c.config.SceneByFragment != nil {
+		s := c.config.getScraper(*c.config.SceneByFragment, c.txnManager, c.globalConfig)
+		return s.scrapeSceneByScene(scene)
+	}
+
+	return nil, nil
+}
+
+func (c *configSceneScraper) scrapeByFragment(scene models.ScrapedSceneInput) (*models.ScrapedScene, error) {
+	if c.config.SceneByQueryFragment != nil {
+		s := c.config.getScraper(*c.config.SceneByQueryFragment, c.txnManager, c.globalConfig)
+		return s.scrapeSceneByFragment(scene)
+	}
+
+	return nil, nil
+}
+
+func (c *configSceneScraper) scrapeByURL(url string) (*models.ScrapedScene, error) {
+	for _, scraper := range c.config.SceneByURL {
+		if scraper.matchesURL(url) {
+			s := c.config.getScraper(scraper.scraperTypeConfig, c.txnManager, c.globalConfig)
+			ret, err := s.scrapeSceneByURL(url)
+			if err != nil {
+				return nil, err
+			}
+
+			if ret != nil {
+				return ret, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+type configPerformerScraper struct {
+	*configScraper
+}
+
+func (c *configPerformerScraper) matchesURL(url string) bool {
+	return c.config.matchesPerformerURL(url)
+}
+
+func (c *configPerformerScraper) scrapeByName(name string) ([]*models.ScrapedPerformer, error) {
+	if c.config.PerformerByName != nil {
+		s := c.config.getScraper(*c.config.PerformerByName, c.txnManager, c.globalConfig)
+		return s.scrapePerformersByName(name)
+	}
+
+	return nil, nil
+}
+
+func (c *configPerformerScraper) scrapeByFragment(scrapedPerformer models.ScrapedPerformerInput) (*models.ScrapedPerformer, error) {
+	if c.config.PerformerByFragment != nil {
+		s := c.config.getScraper(*c.config.PerformerByFragment, c.txnManager, c.globalConfig)
+		return s.scrapePerformerByFragment(scrapedPerformer)
+	}
+
+	// try to match against URL if present
+	if scrapedPerformer.URL != nil && *scrapedPerformer.URL != "" {
+		return c.scrapeByURL(*scrapedPerformer.URL)
+	}
+
+	return nil, nil
+}
+
+func (c *configPerformerScraper) scrapeByURL(url string) (*models.ScrapedPerformer, error) {
+	for _, scraper := range c.config.PerformerByURL {
+		if scraper.matchesURL(url) {
+			s := c.config.getScraper(scraper.scraperTypeConfig, c.txnManager, c.globalConfig)
+			ret, err := s.scrapePerformerByURL(url)
+			if err != nil {
+				return nil, err
+			}
+
+			if ret != nil {
+				return ret, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+type configGalleryScraper struct {
+	*configScraper
+}
+
+func (c *configGalleryScraper) matchesURL(url string) bool {
+	return c.config.matchesGalleryURL(url)
+}
+
+func (c *configGalleryScraper) scrapeByGallery(gallery *models.Gallery) (*models.ScrapedGallery, error) {
+	if c.config.GalleryByFragment != nil {
+		s := c.config.getScraper(*c.config.GalleryByFragment, c.txnManager, c.globalConfig)
+		return s.scrapeGalleryByGallery(gallery)
+	}
+
+	return nil, nil
+}
+
+func (c *configGalleryScraper) scrapeByFragment(gallery models.ScrapedGalleryInput) (*models.ScrapedGallery, error) {
+	if c.config.GalleryByFragment != nil {
+		// TODO - this should be galleryByQueryFragment
+		s := c.config.getScraper(*c.config.GalleryByFragment, c.txnManager, c.globalConfig)
+		return s.scrapeGalleryByFragment(gallery)
+	}
+
+	return nil, nil
+}
+
+func (c *configGalleryScraper) scrapeByURL(url string) (*models.ScrapedGallery, error) {
+	for _, scraper := range c.config.GalleryByURL {
+		if scraper.matchesURL(url) {
+			s := c.config.getScraper(scraper.scraperTypeConfig, c.txnManager, c.globalConfig)
+			ret, err := s.scrapeGalleryByURL(url)
+			if err != nil {
+				return nil, err
+			}
+
+			if ret != nil {
+				return ret, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+type configMovieScraper struct {
+	*configScraper
+}
+
+func (c *configMovieScraper) matchesURL(url string) bool {
+	return c.config.matchesMovieURL(url)
+}
+
+func (c *configMovieScraper) scrapeByURL(url string) (*models.ScrapedMovie, error) {
+	for _, scraper := range c.config.MovieByURL {
+		if scraper.matchesURL(url) {
+			s := c.config.getScraper(scraper.scraperTypeConfig, c.txnManager, c.globalConfig)
+			ret, err := s.scrapeMovieByURL(url)
+			if err != nil {
+				return nil, err
+			}
+
+			if ret != nil {
+				return ret, nil
+			}
+		}
+	}
+
+	return nil, nil
+}
+
+type configScraper struct {
+	config       config
+	txnManager   models.TransactionManager
+	globalConfig GlobalConfig
+}
+
+func createScraperFromConfig(c config, txnManager models.TransactionManager, globalConfig GlobalConfig) scraper {
+	base := configScraper{
+		config:       c,
+		txnManager:   txnManager,
+		globalConfig: globalConfig,
+	}
+
+	ret := scraper{
+		ID:   c.ID,
+		Spec: configScraperSpec(c),
+	}
+
+	// only set fields if supported
+	if c.supportsPerformers() {
+		ret.Performer = &configPerformerScraper{&base}
+	}
+	if c.supportsGalleries() {
+		ret.Gallery = &configGalleryScraper{&base}
+	}
+	if c.supportsMovies() {
+		ret.Movie = &configMovieScraper{&base}
+	}
+	if c.supportsScenes() {
+		ret.Scene = &configSceneScraper{&base}
+	}
+
+	return ret
+}
+
+func configScraperSpec(c config) *models.Scraper {
+	ret := models.Scraper{
+		ID:   c.ID,
+		Name: c.Name,
+	}
+
+	performer := models.ScraperSpec{}
+	if c.PerformerByName != nil {
+		performer.SupportedScrapes = append(performer.SupportedScrapes, models.ScrapeTypeName)
+	}
+	if c.PerformerByFragment != nil {
+		performer.SupportedScrapes = append(performer.SupportedScrapes, models.ScrapeTypeFragment)
+	}
+	if len(c.PerformerByURL) > 0 {
+		performer.SupportedScrapes = append(performer.SupportedScrapes, models.ScrapeTypeURL)
+		for _, v := range c.PerformerByURL {
+			performer.Urls = append(performer.Urls, v.URL...)
+		}
+	}
+
+	if len(performer.SupportedScrapes) > 0 {
+		ret.Performer = &performer
+	}
+
+	scene := models.ScraperSpec{}
+	if c.SceneByFragment != nil {
+		scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeFragment)
+	}
+	if c.SceneByName != nil && c.SceneByQueryFragment != nil {
+		scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeName)
+	}
+	if len(c.SceneByURL) > 0 {
+		scene.SupportedScrapes = append(scene.SupportedScrapes, models.ScrapeTypeURL)
+		for _, v := range c.SceneByURL {
+			scene.Urls = append(scene.Urls, v.URL...)
+		}
+	}
+
+	if len(scene.SupportedScrapes) > 0 {
+		ret.Scene = &scene
+	}
+
+	gallery := models.ScraperSpec{}
+	if c.GalleryByFragment != nil {
+		gallery.SupportedScrapes = append(gallery.SupportedScrapes, models.ScrapeTypeFragment)
+	}
+	if len(c.GalleryByURL) > 0 {
+		gallery.SupportedScrapes = append(gallery.SupportedScrapes, models.ScrapeTypeURL)
+		for _, v := range c.GalleryByURL {
+			gallery.Urls = append(gallery.Urls, v.URL...)
+		}
+	}
+
+	if len(gallery.SupportedScrapes) > 0 {
+		ret.Gallery = &gallery
+	}
+
+	movie := models.ScraperSpec{}
+	if len(c.MovieByURL) > 0 {
+		movie.SupportedScrapes = append(movie.SupportedScrapes, models.ScrapeTypeURL)
+		for _, v := range c.MovieByURL {
+			movie.Urls = append(movie.Urls, v.URL...)
+		}
+	}
+
+	if len(movie.SupportedScrapes) > 0 {
+		ret.Movie = &movie
+	}
+
+	return &ret
+}