mirror of
https://github.com/stashapp/stash.git
synced 2025-12-17 04:14:39 +03:00
Refactor stashbox package (#5699)
* Move stashbox package under pkg * Remove StashBox from method names * Add fingerprint conversion methods to Fingerprint Refactor Fingerprints methods * Make FindSceneByFingerprints accept fingerprints not scene ids * Refactor SubmitSceneDraft to not require readers * Have SubmitFingerprints accept scenes Remove SceneReader dependency * Move ScrapedScene to models package * Move ScrapedImage into models package * Move ScrapedGallery into models package * Move Scene relationship matching out of stashbox package This is now expected to be done in the client code * Remove TagFinder dependency from stashbox.Client * Make stashbox scene find full hierarchy of studios * Move studio resolution into separate method * Move studio matching out of stashbox package This is now client code responsibility * Move performer matching out of FindPerformerByID and FindPerformerByName * Refactor performer querying logic and remove unused stashbox models Renames FindStashBoxPerformersByPerformerNames to QueryPerformers and accepts names instead of performer ids * Refactor SubmitPerformerDraft to not load relationships This will be the responsibility of the calling code * Remove repository references
This commit is contained in:
@@ -29,9 +29,9 @@ type scraperActionImpl interface {
|
||||
scrapeByName(ctx context.Context, name string, ty ScrapeContentType) ([]ScrapedContent, error)
|
||||
scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error)
|
||||
|
||||
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error)
|
||||
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error)
|
||||
scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error)
|
||||
scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error)
|
||||
scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error)
|
||||
scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error)
|
||||
}
|
||||
|
||||
func (c config) getScraper(scraper scraperTypeConfig, client *http.Client, globalConfig GlobalConfig) scraperActionImpl {
|
||||
|
||||
@@ -89,8 +89,8 @@ func autotagMatchTags(ctx context.Context, path string, tagReader models.TagAuto
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
|
||||
var ret *ScrapedScene
|
||||
func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
var ret *models.ScrapedScene
|
||||
const trimExt = false
|
||||
|
||||
// populate performers, studio and tags based on scene path
|
||||
@@ -115,7 +115,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
|
||||
}
|
||||
|
||||
if len(performers) > 0 || studio != nil || len(tags) > 0 {
|
||||
ret = &ScrapedScene{
|
||||
ret = &models.ScrapedScene{
|
||||
Performers: performers,
|
||||
Studio: studio,
|
||||
Tags: tags,
|
||||
@@ -130,7 +130,7 @@ func (s autotagScraper) viaScene(ctx context.Context, _client *http.Client, scen
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
path := gallery.Path
|
||||
if path == "" {
|
||||
// not valid for non-path-based galleries
|
||||
@@ -140,7 +140,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
|
||||
// only trim extension if gallery is file-based
|
||||
trimExt := gallery.PrimaryFileID != nil
|
||||
|
||||
var ret *ScrapedGallery
|
||||
var ret *models.ScrapedGallery
|
||||
|
||||
// populate performers, studio and tags based on scene path
|
||||
if err := txn.WithReadTxn(ctx, s.txnManager, func(ctx context.Context) error {
|
||||
@@ -160,7 +160,7 @@ func (s autotagScraper) viaGallery(ctx context.Context, _client *http.Client, ga
|
||||
}
|
||||
|
||||
if len(performers) > 0 || studio != nil || len(tags) > 0 {
|
||||
ret = &ScrapedGallery{
|
||||
ret = &models.ScrapedGallery{
|
||||
Performers: performers,
|
||||
Studio: studio,
|
||||
Tags: tags,
|
||||
|
||||
@@ -1,32 +0,0 @@
|
||||
package scraper
|
||||
|
||||
import "github.com/stashapp/stash/pkg/models"
|
||||
|
||||
type ScrapedGallery struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Photographer *string `json:"photographer"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
Studio *models.ScrapedStudio `json:"studio"`
|
||||
Tags []*models.ScrapedTag `json:"tags"`
|
||||
Performers []*models.ScrapedPerformer `json:"performers"`
|
||||
|
||||
// deprecated
|
||||
URL *string `json:"url"`
|
||||
}
|
||||
|
||||
func (ScrapedGallery) IsScrapedContent() {}
|
||||
|
||||
type ScrapedGalleryInput struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Photographer *string `json:"photographer"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
|
||||
// deprecated
|
||||
URL *string `json:"url"`
|
||||
}
|
||||
@@ -60,7 +60,7 @@ func (g group) viaFragment(ctx context.Context, client *http.Client, input Input
|
||||
return s.scrapeByFragment(ctx, input)
|
||||
}
|
||||
|
||||
func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error) {
|
||||
func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
if g.config.SceneByFragment == nil {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
@@ -69,7 +69,7 @@ func (g group) viaScene(ctx context.Context, client *http.Client, scene *models.
|
||||
return s.scrapeSceneByScene(ctx, scene)
|
||||
}
|
||||
|
||||
func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
if g.config.GalleryByFragment == nil {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
@@ -78,7 +78,7 @@ func (g group) viaGallery(ctx context.Context, client *http.Client, gallery *mod
|
||||
return s.scrapeGalleryByGallery(ctx, gallery)
|
||||
}
|
||||
|
||||
func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*ScrapedImage, error) {
|
||||
func (g group) viaImage(ctx context.Context, client *http.Client, gallery *models.Image) (*models.ScrapedImage, error) {
|
||||
if g.config.ImageByFragment == nil {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
@@ -11,28 +11,6 @@ import (
|
||||
"github.com/stashapp/stash/pkg/utils"
|
||||
)
|
||||
|
||||
type ScrapedImage struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Photographer *string `json:"photographer"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
Studio *models.ScrapedStudio `json:"studio"`
|
||||
Tags []*models.ScrapedTag `json:"tags"`
|
||||
Performers []*models.ScrapedPerformer `json:"performers"`
|
||||
}
|
||||
|
||||
func (ScrapedImage) IsScrapedContent() {}
|
||||
|
||||
type ScrapedImageInput struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
}
|
||||
|
||||
func setPerformerImage(ctx context.Context, client *http.Client, p *models.ScrapedPerformer, globalConfig GlobalConfig) error {
|
||||
// backwards compatibility: we fetch the image if it's a URL and set it to the first image
|
||||
// Image is deprecated, so only do this if Images is unset
|
||||
@@ -59,7 +37,7 @@ func setPerformerImage(ctx context.Context, client *http.Client, p *models.Scrap
|
||||
return nil
|
||||
}
|
||||
|
||||
func setSceneImage(ctx context.Context, client *http.Client, s *ScrapedScene, globalConfig GlobalConfig) error {
|
||||
func setSceneImage(ctx context.Context, client *http.Client, s *models.ScrapedScene, globalConfig GlobalConfig) error {
|
||||
// don't try to get the image if it doesn't appear to be a URL
|
||||
if s.Image == nil || !strings.HasPrefix(*s.Image, "http") {
|
||||
// nothing to do
|
||||
|
||||
@@ -172,7 +172,7 @@ func (s *jsonScraper) scrapeByName(ctx context.Context, name string, ty ScrapeCo
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
|
||||
func (s *jsonScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromScene(scene)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
@@ -231,7 +231,7 @@ func (s *jsonScraper) scrapeByFragment(ctx context.Context, input Input) (Scrape
|
||||
return scraper.scrapeScene(ctx, q)
|
||||
}
|
||||
|
||||
func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromImage(image)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
@@ -255,7 +255,7 @@ func (s *jsonScraper) scrapeImageByImage(ctx context.Context, image *models.Imag
|
||||
return scraper.scrapeImage(ctx, q)
|
||||
}
|
||||
|
||||
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (s *jsonScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromGallery(gallery)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
|
||||
@@ -997,8 +997,8 @@ func (s mappedScraper) scrapePerformers(ctx context.Context, q mappedQuery) ([]*
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// processSceneRelationships sets the relationships on the ScrapedScene. It returns true if any relationships were set.
|
||||
func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *ScrapedScene) bool {
|
||||
// processSceneRelationships sets the relationships on the models.ScrapedScene. It returns true if any relationships were set.
|
||||
func (s mappedScraper) processSceneRelationships(ctx context.Context, q mappedQuery, resultIndex int, ret *models.ScrapedScene) bool {
|
||||
sceneScraperConfig := s.Scene
|
||||
|
||||
scenePerformersMap := sceneScraperConfig.Performers
|
||||
@@ -1082,8 +1082,8 @@ func processRelationships[T any](ctx context.Context, s mappedScraper, relations
|
||||
return ret
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*ScrapedScene, error) {
|
||||
var ret []*ScrapedScene
|
||||
func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*models.ScrapedScene, error) {
|
||||
var ret []*models.ScrapedScene
|
||||
|
||||
sceneScraperConfig := s.Scene
|
||||
sceneMap := sceneScraperConfig.mappedConfig
|
||||
@@ -1097,7 +1097,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
|
||||
for i, r := range results {
|
||||
logger.Debug(`Processing scene:`)
|
||||
|
||||
var thisScene ScrapedScene
|
||||
var thisScene models.ScrapedScene
|
||||
r.apply(&thisScene)
|
||||
s.processSceneRelationships(ctx, q, i, &thisScene)
|
||||
ret = append(ret, &thisScene)
|
||||
@@ -1106,7 +1106,7 @@ func (s mappedScraper) scrapeScenes(ctx context.Context, q mappedQuery) ([]*Scra
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*ScrapedScene, error) {
|
||||
func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*models.ScrapedScene, error) {
|
||||
sceneScraperConfig := s.Scene
|
||||
if sceneScraperConfig == nil {
|
||||
return nil, nil
|
||||
@@ -1117,7 +1117,7 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
|
||||
logger.Debug(`Processing scene:`)
|
||||
results := sceneMap.process(ctx, q, s.Common, urlsIsMulti)
|
||||
|
||||
var ret ScrapedScene
|
||||
var ret models.ScrapedScene
|
||||
if len(results) > 0 {
|
||||
results[0].apply(&ret)
|
||||
}
|
||||
@@ -1133,8 +1133,8 @@ func (s mappedScraper) scrapeScene(ctx context.Context, q mappedQuery) (*Scraped
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*ScrapedImage, error) {
|
||||
var ret ScrapedImage
|
||||
func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*models.ScrapedImage, error) {
|
||||
var ret models.ScrapedImage
|
||||
|
||||
imageScraperConfig := s.Image
|
||||
if imageScraperConfig == nil {
|
||||
@@ -1184,8 +1184,8 @@ func (s mappedScraper) scrapeImage(ctx context.Context, q mappedQuery) (*Scraped
|
||||
return &ret, nil
|
||||
}
|
||||
|
||||
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*ScrapedGallery, error) {
|
||||
var ret ScrapedGallery
|
||||
func (s mappedScraper) scrapeGallery(ctx context.Context, q mappedQuery) (*models.ScrapedGallery, error) {
|
||||
var ret models.ScrapedGallery
|
||||
|
||||
galleryScraperConfig := s.Gallery
|
||||
if galleryScraperConfig == nil {
|
||||
|
||||
@@ -23,23 +23,23 @@ func (c Cache) postScrape(ctx context.Context, content ScrapedContent, excludeTa
|
||||
}
|
||||
case models.ScrapedPerformer:
|
||||
return c.postScrapePerformer(ctx, v, excludeTagRE)
|
||||
case *ScrapedScene:
|
||||
case *models.ScrapedScene:
|
||||
if v != nil {
|
||||
return c.postScrapeScene(ctx, *v, excludeTagRE)
|
||||
}
|
||||
case ScrapedScene:
|
||||
case models.ScrapedScene:
|
||||
return c.postScrapeScene(ctx, v, excludeTagRE)
|
||||
case *ScrapedGallery:
|
||||
case *models.ScrapedGallery:
|
||||
if v != nil {
|
||||
return c.postScrapeGallery(ctx, *v, excludeTagRE)
|
||||
}
|
||||
case ScrapedGallery:
|
||||
case models.ScrapedGallery:
|
||||
return c.postScrapeGallery(ctx, v, excludeTagRE)
|
||||
case *ScrapedImage:
|
||||
case *models.ScrapedImage:
|
||||
if v != nil {
|
||||
return c.postScrapeImage(ctx, *v, excludeTagRE)
|
||||
}
|
||||
case ScrapedImage:
|
||||
case models.ScrapedImage:
|
||||
return c.postScrapeImage(ctx, v, excludeTagRE)
|
||||
case *models.ScrapedMovie:
|
||||
if v != nil {
|
||||
@@ -133,7 +133,7 @@ func (c Cache) postScrapeMovie(ctx context.Context, m models.ScrapedMovie, exclu
|
||||
m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
|
||||
|
||||
if m.Studio != nil {
|
||||
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
|
||||
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -165,7 +165,7 @@ func (c Cache) postScrapeGroup(ctx context.Context, m models.ScrapedGroup, exclu
|
||||
m.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
|
||||
|
||||
if m.Studio != nil {
|
||||
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, nil); err != nil {
|
||||
if err := match.ScrapedStudio(ctx, r.StudioFinder, m.Studio, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -201,7 +201,7 @@ func (c Cache) postScrapeScenePerformer(ctx context.Context, p models.ScrapedPer
|
||||
return ignoredTags, nil
|
||||
}
|
||||
|
||||
func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
func (c Cache) postScrapeScene(ctx context.Context, scene models.ScrapedScene, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
// set the URL/URLs field
|
||||
if scene.URL == nil && len(scene.URLs) > 0 {
|
||||
scene.URL = &scene.URLs[0]
|
||||
@@ -227,7 +227,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
|
||||
return err
|
||||
}
|
||||
|
||||
if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
|
||||
if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -277,7 +277,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
|
||||
scene.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
|
||||
|
||||
if scene.Studio != nil {
|
||||
err := match.ScrapedStudio(ctx, sqb, scene.Studio, nil)
|
||||
err := match.ScrapedStudio(ctx, sqb, scene.Studio, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -296,7 +296,7 @@ func (c Cache) postScrapeScene(ctx context.Context, scene ScrapedScene, excludeT
|
||||
return scene, ignoredTags, nil
|
||||
}
|
||||
|
||||
func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
func (c Cache) postScrapeGallery(ctx context.Context, g models.ScrapedGallery, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
// set the URL/URLs field
|
||||
if g.URL == nil && len(g.URLs) > 0 {
|
||||
g.URL = &g.URLs[0]
|
||||
@@ -312,7 +312,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
|
||||
sqb := r.StudioFinder
|
||||
|
||||
for _, p := range g.Performers {
|
||||
err := match.ScrapedPerformer(ctx, pqb, p, nil)
|
||||
err := match.ScrapedPerformer(ctx, pqb, p, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -325,7 +325,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
|
||||
g.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
|
||||
|
||||
if g.Studio != nil {
|
||||
err := match.ScrapedStudio(ctx, sqb, g.Studio, nil)
|
||||
err := match.ScrapedStudio(ctx, sqb, g.Studio, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -339,7 +339,7 @@ func (c Cache) postScrapeGallery(ctx context.Context, g ScrapedGallery, excludeT
|
||||
return g, ignoredTags, nil
|
||||
}
|
||||
|
||||
func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
func (c Cache) postScrapeImage(ctx context.Context, image models.ScrapedImage, excludeTagRE []*regexp.Regexp) (_ ScrapedContent, ignoredTags []string, err error) {
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
pqb := r.PerformerFinder
|
||||
@@ -347,7 +347,7 @@ func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeT
|
||||
sqb := r.StudioFinder
|
||||
|
||||
for _, p := range image.Performers {
|
||||
if err := match.ScrapedPerformer(ctx, pqb, p, nil); err != nil {
|
||||
if err := match.ScrapedPerformer(ctx, pqb, p, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -360,7 +360,7 @@ func (c Cache) postScrapeImage(ctx context.Context, image ScrapedImage, excludeT
|
||||
image.Tags, ignoredTags = FilterTags(excludeTagRE, tags)
|
||||
|
||||
if image.Studio != nil {
|
||||
err := match.ScrapedStudio(ctx, sqb, image.Studio, nil)
|
||||
err := match.ScrapedStudio(ctx, sqb, image.Studio, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ func queryURLParametersFromScene(scene *models.Scene) queryURLParameters {
|
||||
return ret
|
||||
}
|
||||
|
||||
func queryURLParametersFromScrapedScene(scene ScrapedSceneInput) queryURLParameters {
|
||||
func queryURLParametersFromScrapedScene(scene models.ScrapedSceneInput) queryURLParameters {
|
||||
ret := make(queryURLParameters)
|
||||
|
||||
setField := func(field string, value *string) {
|
||||
|
||||
@@ -1,39 +0,0 @@
|
||||
package scraper
|
||||
|
||||
import (
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
)
|
||||
|
||||
type ScrapedScene struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Director *string `json:"director"`
|
||||
URL *string `json:"url"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
// This should be a base64 encoded data URL
|
||||
Image *string `json:"image"`
|
||||
File *models.SceneFileType `json:"file"`
|
||||
Studio *models.ScrapedStudio `json:"studio"`
|
||||
Tags []*models.ScrapedTag `json:"tags"`
|
||||
Performers []*models.ScrapedPerformer `json:"performers"`
|
||||
Groups []*models.ScrapedGroup `json:"groups"`
|
||||
Movies []*models.ScrapedMovie `json:"movies"`
|
||||
RemoteSiteID *string `json:"remote_site_id"`
|
||||
Duration *int `json:"duration"`
|
||||
Fingerprints []*models.StashBoxFingerprint `json:"fingerprints"`
|
||||
}
|
||||
|
||||
func (ScrapedScene) IsScrapedContent() {}
|
||||
|
||||
type ScrapedSceneInput struct {
|
||||
Title *string `json:"title"`
|
||||
Code *string `json:"code"`
|
||||
Details *string `json:"details"`
|
||||
Director *string `json:"director"`
|
||||
URL *string `json:"url"`
|
||||
URLs []string `json:"urls"`
|
||||
Date *string `json:"date"`
|
||||
RemoteSiteID *string `json:"remote_site_id"`
|
||||
}
|
||||
@@ -163,9 +163,9 @@ var (
|
||||
// set to nil.
|
||||
type Input struct {
|
||||
Performer *ScrapedPerformerInput
|
||||
Scene *ScrapedSceneInput
|
||||
Gallery *ScrapedGalleryInput
|
||||
Image *ScrapedImageInput
|
||||
Scene *models.ScrapedSceneInput
|
||||
Gallery *models.ScrapedGalleryInput
|
||||
Image *models.ScrapedImageInput
|
||||
}
|
||||
|
||||
// populateURL populates the URL field of the input based on the
|
||||
@@ -227,7 +227,7 @@ type fragmentScraper interface {
|
||||
type sceneScraper interface {
|
||||
scraper
|
||||
|
||||
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*ScrapedScene, error)
|
||||
viaScene(ctx context.Context, client *http.Client, scene *models.Scene) (*models.ScrapedScene, error)
|
||||
}
|
||||
|
||||
// imageScraper is a scraper which supports image scrapes with
|
||||
@@ -235,7 +235,7 @@ type sceneScraper interface {
|
||||
type imageScraper interface {
|
||||
scraper
|
||||
|
||||
viaImage(ctx context.Context, client *http.Client, image *models.Image) (*ScrapedImage, error)
|
||||
viaImage(ctx context.Context, client *http.Client, image *models.Image) (*models.ScrapedImage, error)
|
||||
}
|
||||
|
||||
// galleryScraper is a scraper which supports gallery scrapes with
|
||||
@@ -243,5 +243,5 @@ type imageScraper interface {
|
||||
type galleryScraper interface {
|
||||
scraper
|
||||
|
||||
viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*ScrapedGallery, error)
|
||||
viaGallery(ctx context.Context, client *http.Client, gallery *models.Gallery) (*models.ScrapedGallery, error)
|
||||
}
|
||||
|
||||
@@ -328,7 +328,7 @@ func (s *scriptScraper) scrapeByName(ctx context.Context, name string, ty Scrape
|
||||
}
|
||||
}
|
||||
case ScrapeContentTypeScene:
|
||||
var scenes []ScrapedScene
|
||||
var scenes []models.ScrapedScene
|
||||
err = s.runScraperScript(ctx, input, &scenes)
|
||||
if err == nil {
|
||||
for _, s := range scenes {
|
||||
@@ -377,11 +377,11 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
|
||||
err := s.runScraperScript(ctx, input, &performer)
|
||||
return performer, err
|
||||
case ScrapeContentTypeGallery:
|
||||
var gallery *ScrapedGallery
|
||||
var gallery *models.ScrapedGallery
|
||||
err := s.runScraperScript(ctx, input, &gallery)
|
||||
return gallery, err
|
||||
case ScrapeContentTypeScene:
|
||||
var scene *ScrapedScene
|
||||
var scene *models.ScrapedScene
|
||||
err := s.runScraperScript(ctx, input, &scene)
|
||||
return scene, err
|
||||
case ScrapeContentTypeMovie, ScrapeContentTypeGroup:
|
||||
@@ -389,7 +389,7 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
|
||||
err := s.runScraperScript(ctx, input, &movie)
|
||||
return movie, err
|
||||
case ScrapeContentTypeImage:
|
||||
var image *ScrapedImage
|
||||
var image *models.ScrapedImage
|
||||
err := s.runScraperScript(ctx, input, &image)
|
||||
return image, err
|
||||
}
|
||||
@@ -397,42 +397,42 @@ func (s *scriptScraper) scrape(ctx context.Context, input string, ty ScrapeConte
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
|
||||
func (s *scriptScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
inString, err := json.Marshal(sceneInputFromScene(scene))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *ScrapedScene
|
||||
var ret *models.ScrapedScene
|
||||
|
||||
err = s.runScraperScript(ctx, string(inString), &ret)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (s *scriptScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
inString, err := json.Marshal(galleryInputFromGallery(gallery))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *ScrapedGallery
|
||||
var ret *models.ScrapedGallery
|
||||
|
||||
err = s.runScraperScript(ctx, string(inString), &ret)
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
func (s *scriptScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
|
||||
inString, err := json.Marshal(imageToUpdateInput(image))
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *ScrapedImage
|
||||
var ret *models.ScrapedImage
|
||||
|
||||
err = s.runScraperScript(ctx, string(inString), &ret)
|
||||
|
||||
|
||||
@@ -216,11 +216,11 @@ type scrapedStudioStash struct {
|
||||
|
||||
type stashFindSceneNamesResultType struct {
|
||||
Count int `graphql:"count"`
|
||||
Scenes []*scrapedSceneStash `graphql:"scenes"`
|
||||
Scenes []*ScrapedSceneStash `graphql:"scenes"`
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *scrapedSceneStash) (*ScrapedScene, error) {
|
||||
ret := ScrapedScene{}
|
||||
func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scene *ScrapedSceneStash) (*models.ScrapedScene, error) {
|
||||
ret := models.ScrapedScene{}
|
||||
err := copier.Copy(&ret, scene)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -329,7 +329,7 @@ func (f stashVideoFile) SceneFileType() models.SceneFileType {
|
||||
return ret
|
||||
}
|
||||
|
||||
type scrapedSceneStash struct {
|
||||
type ScrapedSceneStash struct {
|
||||
ID string `graphql:"id" json:"id"`
|
||||
Title *string `graphql:"title" json:"title"`
|
||||
Details *string `graphql:"details" json:"details"`
|
||||
@@ -341,10 +341,10 @@ type scrapedSceneStash struct {
|
||||
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
|
||||
func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
// query by MD5
|
||||
var q struct {
|
||||
FindScene *scrapedSceneStash `graphql:"findSceneByHash(input: $c)"`
|
||||
FindScene *ScrapedSceneStash `graphql:"findSceneByHash(input: $c)"`
|
||||
}
|
||||
|
||||
type SceneHashInput struct {
|
||||
@@ -401,7 +401,7 @@ type scrapedGalleryStash struct {
|
||||
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
var q struct {
|
||||
FindGallery *scrapedGalleryStash `graphql:"findGalleryByHash(input: $c)"`
|
||||
}
|
||||
@@ -425,7 +425,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
||||
}
|
||||
|
||||
// need to copy back to a scraped scene
|
||||
ret := ScrapedGallery{}
|
||||
ret := models.ScrapedGallery{}
|
||||
if err := copier.Copy(&ret, q.FindGallery); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -433,7 +433,7 @@ func (s *stashScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
||||
return &ret, nil
|
||||
}
|
||||
|
||||
func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
func (s *stashScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
// Package stashbox provides a client interface to a stash-box server instance.
|
||||
package stashbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"regexp"
|
||||
|
||||
"github.com/Yamashou/gqlgenc/clientv2"
|
||||
"github.com/stashapp/stash/pkg/match"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/scraper"
|
||||
"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
|
||||
"github.com/stashapp/stash/pkg/txn"
|
||||
)
|
||||
|
||||
type SceneReader interface {
|
||||
models.SceneGetter
|
||||
models.StashIDLoader
|
||||
models.VideoFileLoader
|
||||
}
|
||||
|
||||
type PerformerReader interface {
|
||||
models.PerformerGetter
|
||||
match.PerformerFinder
|
||||
models.AliasLoader
|
||||
models.StashIDLoader
|
||||
models.URLLoader
|
||||
FindBySceneID(ctx context.Context, sceneID int) ([]*models.Performer, error)
|
||||
GetImage(ctx context.Context, performerID int) ([]byte, error)
|
||||
}
|
||||
|
||||
type StudioReader interface {
|
||||
models.StudioGetter
|
||||
match.StudioFinder
|
||||
models.StashIDLoader
|
||||
}
|
||||
|
||||
type TagFinder interface {
|
||||
models.TagQueryer
|
||||
FindBySceneID(ctx context.Context, sceneID int) ([]*models.Tag, error)
|
||||
}
|
||||
|
||||
type Repository struct {
|
||||
TxnManager models.TxnManager
|
||||
|
||||
Scene SceneReader
|
||||
Performer PerformerReader
|
||||
Tag TagFinder
|
||||
Studio StudioReader
|
||||
}
|
||||
|
||||
func NewRepository(repo models.Repository) Repository {
|
||||
return Repository{
|
||||
TxnManager: repo.TxnManager,
|
||||
Scene: repo.Scene,
|
||||
Performer: repo.Performer,
|
||||
Tag: repo.Tag,
|
||||
Studio: repo.Studio,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Repository) WithReadTxn(ctx context.Context, fn txn.TxnFunc) error {
|
||||
return txn.WithReadTxn(ctx, r.TxnManager, fn)
|
||||
}
|
||||
|
||||
// Client represents the client interface to a stash-box server instance.
|
||||
type Client struct {
|
||||
client *graphql.Client
|
||||
repository Repository
|
||||
box models.StashBox
|
||||
|
||||
// tag patterns to be excluded
|
||||
excludeTagRE []*regexp.Regexp
|
||||
}
|
||||
|
||||
// NewClient returns a new instance of a stash-box client.
|
||||
func NewClient(box models.StashBox, repo Repository, excludeTagPatterns []string) *Client {
|
||||
authHeader := func(ctx context.Context, req *http.Request, gqlInfo *clientv2.GQLRequestInfo, res interface{}, next clientv2.RequestInterceptorFunc) error {
|
||||
req.Header.Set("ApiKey", box.APIKey)
|
||||
return next(ctx, req, gqlInfo, res)
|
||||
}
|
||||
|
||||
client := &graphql.Client{
|
||||
Client: clientv2.NewClient(http.DefaultClient, box.Endpoint, nil, authHeader),
|
||||
}
|
||||
|
||||
return &Client{
|
||||
client: client,
|
||||
repository: repo,
|
||||
box: box,
|
||||
excludeTagRE: scraper.CompileExclusionRegexps(excludeTagPatterns),
|
||||
}
|
||||
}
|
||||
|
||||
func (c Client) getHTTPClient() *http.Client {
|
||||
return c.client.Client.Client
|
||||
}
|
||||
|
||||
func (c Client) GetUser(ctx context.Context) (*graphql.Me, error) {
|
||||
return c.client.Me(ctx)
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
package stashbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
|
||||
"github.com/Yamashou/gqlgenc/clientv2"
|
||||
"github.com/Yamashou/gqlgenc/graphqljson"
|
||||
)
|
||||
|
||||
func (c *Client) submitDraft(ctx context.Context, query string, input interface{}, image io.Reader, ret interface{}) error {
|
||||
vars := map[string]interface{}{
|
||||
"input": input,
|
||||
}
|
||||
|
||||
r := &clientv2.Request{
|
||||
Query: query,
|
||||
Variables: vars,
|
||||
OperationName: "",
|
||||
}
|
||||
|
||||
requestBody, err := json.Marshal(r)
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode: %w", err)
|
||||
}
|
||||
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
if err := writer.WriteField("operations", string(requestBody)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if image != nil {
|
||||
if err := writer.WriteField("map", "{ \"0\": [\"variables.input.image\"] }"); err != nil {
|
||||
return err
|
||||
}
|
||||
part, _ := writer.CreateFormFile("0", "draft")
|
||||
if _, err := io.Copy(part, image); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if err := writer.WriteField("map", "{}"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
writer.Close()
|
||||
|
||||
req, _ := http.NewRequestWithContext(ctx, "POST", c.box.Endpoint, body)
|
||||
req.Header.Add("Content-Type", writer.FormDataContentType())
|
||||
req.Header.Set("ApiKey", c.box.APIKey)
|
||||
|
||||
httpClient := c.client.Client.Client
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
responseBytes, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Data json.RawMessage `json:"data"`
|
||||
Errors json.RawMessage `json:"errors"`
|
||||
}
|
||||
|
||||
var respGQL response
|
||||
|
||||
if err := json.Unmarshal(responseBytes, &respGQL); err != nil {
|
||||
return fmt.Errorf("failed to decode data %s: %w", string(responseBytes), err)
|
||||
}
|
||||
|
||||
if len(respGQL.Errors) > 0 {
|
||||
// try to parse standard graphql error
|
||||
errors := &clientv2.GqlErrorList{}
|
||||
if e := json.Unmarshal(responseBytes, errors); e != nil {
|
||||
return fmt.Errorf("failed to parse graphql errors. Response content %s - %w ", string(responseBytes), e)
|
||||
}
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
if err := graphqljson.UnmarshalData(respGQL.Data, ret); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// we can't currently use this due to https://github.com/Yamashou/gqlgenc/issues/109
|
||||
// func uploadImage(image io.Reader) client.HTTPRequestOption {
|
||||
// return func(req *http.Request) {
|
||||
// if image == nil {
|
||||
// // return without changing anything
|
||||
// return
|
||||
// }
|
||||
|
||||
// // we can't handle errors in here, so if one happens, just return
|
||||
// // without changing anything.
|
||||
|
||||
// // repackage the request to include the image
|
||||
// bodyBytes, err := ioutil.ReadAll(req.Body)
|
||||
// if err != nil {
|
||||
// return
|
||||
// }
|
||||
|
||||
// newBody := &bytes.Buffer{}
|
||||
// writer := multipart.NewWriter(newBody)
|
||||
// _ = writer.WriteField("operations", string(bodyBytes))
|
||||
|
||||
// if err := writer.WriteField("map", "{ \"0\": [\"variables.input.image\"] }"); err != nil {
|
||||
// return
|
||||
// }
|
||||
// part, _ := writer.CreateFormFile("0", "draft")
|
||||
// if _, err := io.Copy(part, image); err != nil {
|
||||
// return
|
||||
// }
|
||||
|
||||
// writer.Close()
|
||||
|
||||
// // now set the request body to this new body
|
||||
// req.Body = io.NopCloser(newBody)
|
||||
// req.ContentLength = int64(newBody.Len())
|
||||
// req.Header.Set("Content-Type", writer.FormDataContentType())
|
||||
// }
|
||||
// }
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,20 +0,0 @@
|
||||
package graphql
|
||||
|
||||
import "github.com/99designs/gqlgen/graphql"
|
||||
|
||||
// Override for generated struct due to mistaken omitempty
|
||||
// https://github.com/Yamashou/gqlgenc/issues/77
|
||||
type SceneDraftInput struct {
|
||||
ID *string `json:"id,omitempty"`
|
||||
Title *string `json:"title,omitempty"`
|
||||
Code *string `json:"code,omitempty"`
|
||||
Details *string `json:"details,omitempty"`
|
||||
Director *string `json:"director,omitempty"`
|
||||
URL *string `json:"url,omitempty"`
|
||||
Date *string `json:"date,omitempty"`
|
||||
Studio *DraftEntityInput `json:"studio,omitempty"`
|
||||
Performers []*DraftEntityInput `json:"performers"`
|
||||
Tags []*DraftEntityInput `json:"tags,omitempty"`
|
||||
Image *graphql.Upload `json:"image,omitempty"`
|
||||
Fingerprints []*FingerprintInput `json:"fingerprints"`
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
package stashbox
|
||||
|
||||
import "github.com/stashapp/stash/pkg/models"
|
||||
|
||||
type StashBoxStudioQueryResult struct {
|
||||
Query string `json:"query"`
|
||||
Results []*models.ScrapedStudio `json:"results"`
|
||||
}
|
||||
|
||||
type StashBoxPerformerQueryResult struct {
|
||||
Query string `json:"query"`
|
||||
Results []*models.ScrapedPerformer `json:"results"`
|
||||
}
|
||||
@@ -1,555 +0,0 @@
|
||||
package stashbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/stashapp/stash/pkg/match"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/scraper"
|
||||
"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
|
||||
"github.com/stashapp/stash/pkg/sliceutil"
|
||||
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
|
||||
"github.com/stashapp/stash/pkg/utils"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
)
|
||||
|
||||
// QueryStashBoxPerformer queries stash-box for performers using a query string.
|
||||
func (c Client) QueryStashBoxPerformer(ctx context.Context, queryStr string) ([]*StashBoxPerformerQueryResult, error) {
|
||||
performers, err := c.queryStashBoxPerformer(ctx, queryStr)
|
||||
|
||||
res := []*StashBoxPerformerQueryResult{
|
||||
{
|
||||
Query: queryStr,
|
||||
Results: performers,
|
||||
},
|
||||
}
|
||||
|
||||
// set the deprecated image field
|
||||
for _, p := range res[0].Results {
|
||||
if len(p.Images) > 0 {
|
||||
p.Image = &p.Images[0]
|
||||
}
|
||||
}
|
||||
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (c Client) queryStashBoxPerformer(ctx context.Context, queryStr string) ([]*models.ScrapedPerformer, error) {
|
||||
performers, err := c.client.SearchPerformer(ctx, queryStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
performerFragments := performers.SearchPerformer
|
||||
|
||||
var ret []*models.ScrapedPerformer
|
||||
var ignoredTags []string
|
||||
for _, fragment := range performerFragments {
|
||||
performer := performerFragmentToScrapedPerformer(*fragment)
|
||||
|
||||
// exclude tags that match the excludeTagRE
|
||||
var thisIgnoredTags []string
|
||||
performer.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, performer.Tags)
|
||||
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
|
||||
|
||||
ret = append(ret, performer)
|
||||
}
|
||||
|
||||
scraper.LogIgnoredTags(ignoredTags)
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// FindStashBoxPerformersByNames queries stash-box for performers by name
|
||||
func (c Client) FindStashBoxPerformersByNames(ctx context.Context, performerIDs []string) ([]*StashBoxPerformerQueryResult, error) {
|
||||
ids, err := stringslice.StringSliceToIntSlice(performerIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var performers []*models.Performer
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
qb := r.Performer
|
||||
|
||||
for _, performerID := range ids {
|
||||
performer, err := qb.Find(ctx, performerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if performer == nil {
|
||||
return fmt.Errorf("performer with id %d not found", performerID)
|
||||
}
|
||||
|
||||
if performer.Name != "" {
|
||||
performers = append(performers, performer)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c.findStashBoxPerformersByNames(ctx, performers)
|
||||
}
|
||||
|
||||
func (c Client) FindStashBoxPerformersByPerformerNames(ctx context.Context, performerIDs []string) ([][]*models.ScrapedPerformer, error) {
|
||||
ids, err := stringslice.StringSliceToIntSlice(performerIDs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var performers []*models.Performer
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
qb := r.Performer
|
||||
|
||||
for _, performerID := range ids {
|
||||
performer, err := qb.Find(ctx, performerID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if performer == nil {
|
||||
return fmt.Errorf("performer with id %d not found", performerID)
|
||||
}
|
||||
|
||||
if performer.Name != "" {
|
||||
performers = append(performers, performer)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
results, err := c.findStashBoxPerformersByNames(ctx, performers)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret [][]*models.ScrapedPerformer
|
||||
for _, r := range results {
|
||||
ret = append(ret, r.Results)
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (c Client) findStashBoxPerformersByNames(ctx context.Context, performers []*models.Performer) ([]*StashBoxPerformerQueryResult, error) {
|
||||
var ret []*StashBoxPerformerQueryResult
|
||||
for _, performer := range performers {
|
||||
if performer.Name != "" {
|
||||
performerResults, err := c.queryStashBoxPerformer(ctx, performer.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := StashBoxPerformerQueryResult{
|
||||
Query: strconv.Itoa(performer.ID),
|
||||
Results: performerResults,
|
||||
}
|
||||
|
||||
ret = append(ret, &result)
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func findURL(urls []*graphql.URLFragment, urlType string) *string {
|
||||
for _, u := range urls {
|
||||
if u.Type == urlType {
|
||||
ret := u.URL
|
||||
return &ret
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func enumToStringPtr(e fmt.Stringer, titleCase bool) *string {
|
||||
if e != nil {
|
||||
ret := strings.ReplaceAll(e.String(), "_", " ")
|
||||
if titleCase {
|
||||
c := cases.Title(language.Und)
|
||||
ret = c.String(strings.ToLower(ret))
|
||||
}
|
||||
return &ret
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func translateGender(gender *graphql.GenderEnum) *string {
|
||||
var res models.GenderEnum
|
||||
switch *gender {
|
||||
case graphql.GenderEnumMale:
|
||||
res = models.GenderEnumMale
|
||||
case graphql.GenderEnumFemale:
|
||||
res = models.GenderEnumFemale
|
||||
case graphql.GenderEnumIntersex:
|
||||
res = models.GenderEnumIntersex
|
||||
case graphql.GenderEnumTransgenderFemale:
|
||||
res = models.GenderEnumTransgenderFemale
|
||||
case graphql.GenderEnumTransgenderMale:
|
||||
res = models.GenderEnumTransgenderMale
|
||||
case graphql.GenderEnumNonBinary:
|
||||
res = models.GenderEnumNonBinary
|
||||
}
|
||||
|
||||
if res != "" {
|
||||
strVal := res.String()
|
||||
return &strVal
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatMeasurements(m graphql.MeasurementsFragment) *string {
|
||||
if m.BandSize != nil && m.CupSize != nil && m.Hip != nil && m.Waist != nil {
|
||||
ret := fmt.Sprintf("%d%s-%d-%d", *m.BandSize, *m.CupSize, *m.Waist, *m.Hip)
|
||||
return &ret
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func formatCareerLength(start, end *int) *string {
|
||||
if start == nil && end == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var ret string
|
||||
switch {
|
||||
case end == nil:
|
||||
ret = fmt.Sprintf("%d -", *start)
|
||||
case start == nil:
|
||||
ret = fmt.Sprintf("- %d", *end)
|
||||
default:
|
||||
ret = fmt.Sprintf("%d - %d", *start, *end)
|
||||
}
|
||||
|
||||
return &ret
|
||||
}
|
||||
|
||||
func formatBodyModifications(m []*graphql.BodyModificationFragment) *string {
|
||||
if len(m) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var retSlice []string
|
||||
for _, f := range m {
|
||||
if f.Description == nil {
|
||||
retSlice = append(retSlice, f.Location)
|
||||
} else {
|
||||
retSlice = append(retSlice, fmt.Sprintf("%s, %s", f.Location, *f.Description))
|
||||
}
|
||||
}
|
||||
|
||||
ret := strings.Join(retSlice, "; ")
|
||||
return &ret
|
||||
}
|
||||
|
||||
func fetchImage(ctx context.Context, client *http.Client, url string) (*string, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := client.Do(req)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// determine the image type and set the base64 type
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = http.DetectContentType(body)
|
||||
}
|
||||
|
||||
img := "data:" + contentType + ";base64," + utils.GetBase64StringFromData(body)
|
||||
return &img, nil
|
||||
}
|
||||
|
||||
func performerFragmentToScrapedPerformer(p graphql.PerformerFragment) *models.ScrapedPerformer {
|
||||
images := []string{}
|
||||
for _, image := range p.Images {
|
||||
images = append(images, image.URL)
|
||||
}
|
||||
|
||||
sp := &models.ScrapedPerformer{
|
||||
Name: &p.Name,
|
||||
Disambiguation: p.Disambiguation,
|
||||
Country: p.Country,
|
||||
Measurements: formatMeasurements(*p.Measurements),
|
||||
CareerLength: formatCareerLength(p.CareerStartYear, p.CareerEndYear),
|
||||
Tattoos: formatBodyModifications(p.Tattoos),
|
||||
Piercings: formatBodyModifications(p.Piercings),
|
||||
Twitter: findURL(p.Urls, "TWITTER"),
|
||||
RemoteSiteID: &p.ID,
|
||||
RemoteDeleted: p.Deleted,
|
||||
RemoteMergedIntoId: p.MergedIntoID,
|
||||
Images: images,
|
||||
// TODO - tags not currently supported
|
||||
// graphql schema change to accommodate this. Leave off for now.
|
||||
}
|
||||
|
||||
if len(sp.Images) > 0 {
|
||||
sp.Image = &sp.Images[0]
|
||||
}
|
||||
|
||||
if p.Height != nil && *p.Height > 0 {
|
||||
hs := strconv.Itoa(*p.Height)
|
||||
sp.Height = &hs
|
||||
}
|
||||
|
||||
if p.BirthDate != nil {
|
||||
sp.Birthdate = padFuzzyDate(p.BirthDate)
|
||||
}
|
||||
|
||||
if p.DeathDate != nil {
|
||||
sp.DeathDate = padFuzzyDate(p.DeathDate)
|
||||
}
|
||||
|
||||
if p.Gender != nil {
|
||||
sp.Gender = translateGender(p.Gender)
|
||||
}
|
||||
|
||||
if p.Ethnicity != nil {
|
||||
sp.Ethnicity = enumToStringPtr(p.Ethnicity, true)
|
||||
}
|
||||
|
||||
if p.EyeColor != nil {
|
||||
sp.EyeColor = enumToStringPtr(p.EyeColor, true)
|
||||
}
|
||||
|
||||
if p.HairColor != nil {
|
||||
sp.HairColor = enumToStringPtr(p.HairColor, true)
|
||||
}
|
||||
|
||||
if p.BreastType != nil {
|
||||
sp.FakeTits = enumToStringPtr(p.BreastType, true)
|
||||
}
|
||||
|
||||
if len(p.Aliases) > 0 {
|
||||
// #4437 - stash-box may return aliases that are equal to the performer name
|
||||
// filter these out
|
||||
p.Aliases = sliceutil.Filter(p.Aliases, func(s string) bool {
|
||||
return !strings.EqualFold(s, p.Name)
|
||||
})
|
||||
|
||||
// #4596 - stash-box may return duplicate aliases. Filter these out
|
||||
p.Aliases = stringslice.UniqueFold(p.Aliases)
|
||||
|
||||
alias := strings.Join(p.Aliases, ", ")
|
||||
sp.Aliases = &alias
|
||||
}
|
||||
|
||||
for _, u := range p.Urls {
|
||||
sp.URLs = append(sp.URLs, u.URL)
|
||||
}
|
||||
|
||||
return sp
|
||||
}
|
||||
|
||||
func padFuzzyDate(date *string) *string {
|
||||
if date == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var paddedDate string
|
||||
switch len(*date) {
|
||||
case 10:
|
||||
paddedDate = *date
|
||||
case 7:
|
||||
paddedDate = fmt.Sprintf("%s-01", *date)
|
||||
case 4:
|
||||
paddedDate = fmt.Sprintf("%s-01-01", *date)
|
||||
}
|
||||
return &paddedDate
|
||||
}
|
||||
|
||||
func (c Client) FindStashBoxPerformerByID(ctx context.Context, id string) (*models.ScrapedPerformer, error) {
|
||||
performer, err := c.client.FindPerformerByID(ctx, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if performer.FindPerformer == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
ret := performerFragmentToScrapedPerformer(*performer.FindPerformer)
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
err := match.ScrapedPerformer(ctx, r.Performer, ret, &c.box.Endpoint)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (c Client) FindStashBoxPerformerByName(ctx context.Context, name string) (*models.ScrapedPerformer, error) {
|
||||
performers, err := c.client.SearchPerformer(ctx, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *models.ScrapedPerformer
|
||||
for _, performer := range performers.SearchPerformer {
|
||||
if strings.EqualFold(performer.Name, name) {
|
||||
ret = performerFragmentToScrapedPerformer(*performer)
|
||||
}
|
||||
}
|
||||
|
||||
if ret == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
err := match.ScrapedPerformer(ctx, r.Performer, ret, &c.box.Endpoint)
|
||||
return err
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (c Client) SubmitPerformerDraft(ctx context.Context, performer *models.Performer) (*string, error) {
|
||||
draft := graphql.PerformerDraftInput{}
|
||||
var image io.Reader
|
||||
pqb := c.repository.Performer
|
||||
endpoint := c.box.Endpoint
|
||||
|
||||
if err := performer.LoadAliases(ctx, pqb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := performer.LoadURLs(ctx, pqb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := performer.LoadStashIDs(ctx, pqb); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
img, _ := pqb.GetImage(ctx, performer.ID)
|
||||
if img != nil {
|
||||
image = bytes.NewReader(img)
|
||||
}
|
||||
|
||||
if performer.Name != "" {
|
||||
draft.Name = performer.Name
|
||||
}
|
||||
if performer.Disambiguation != "" {
|
||||
draft.Disambiguation = &performer.Disambiguation
|
||||
}
|
||||
if performer.Birthdate != nil {
|
||||
d := performer.Birthdate.String()
|
||||
draft.Birthdate = &d
|
||||
}
|
||||
if performer.Country != "" {
|
||||
draft.Country = &performer.Country
|
||||
}
|
||||
if performer.Ethnicity != "" {
|
||||
draft.Ethnicity = &performer.Ethnicity
|
||||
}
|
||||
if performer.EyeColor != "" {
|
||||
draft.EyeColor = &performer.EyeColor
|
||||
}
|
||||
if performer.FakeTits != "" {
|
||||
draft.BreastType = &performer.FakeTits
|
||||
}
|
||||
if performer.Gender != nil && performer.Gender.IsValid() {
|
||||
v := performer.Gender.String()
|
||||
draft.Gender = &v
|
||||
}
|
||||
if performer.HairColor != "" {
|
||||
draft.HairColor = &performer.HairColor
|
||||
}
|
||||
if performer.Height != nil {
|
||||
v := strconv.Itoa(*performer.Height)
|
||||
draft.Height = &v
|
||||
}
|
||||
if performer.Measurements != "" {
|
||||
draft.Measurements = &performer.Measurements
|
||||
}
|
||||
if performer.Piercings != "" {
|
||||
draft.Piercings = &performer.Piercings
|
||||
}
|
||||
if performer.Tattoos != "" {
|
||||
draft.Tattoos = &performer.Tattoos
|
||||
}
|
||||
if len(performer.Aliases.List()) > 0 {
|
||||
aliases := strings.Join(performer.Aliases.List(), ",")
|
||||
draft.Aliases = &aliases
|
||||
}
|
||||
if performer.CareerLength != "" {
|
||||
var career = strings.Split(performer.CareerLength, "-")
|
||||
if i, err := strconv.Atoi(strings.TrimSpace(career[0])); err == nil {
|
||||
draft.CareerStartYear = &i
|
||||
}
|
||||
if len(career) == 2 {
|
||||
if y, err := strconv.Atoi(strings.TrimSpace(career[1])); err == nil {
|
||||
draft.CareerEndYear = &y
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if len(performer.URLs.List()) > 0 {
|
||||
draft.Urls = performer.URLs.List()
|
||||
}
|
||||
|
||||
stashIDs, err := pqb.GetStashIDs(ctx, performer.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var stashID *string
|
||||
for _, v := range stashIDs {
|
||||
c := v
|
||||
if v.Endpoint == endpoint {
|
||||
stashID = &c.StashID
|
||||
break
|
||||
}
|
||||
}
|
||||
draft.ID = stashID
|
||||
|
||||
var id *string
|
||||
var ret graphql.SubmitPerformerDraft
|
||||
err = c.submitDraft(ctx, graphql.SubmitPerformerDraftDocument, draft, image, &ret)
|
||||
id = ret.SubmitPerformerDraft.ID
|
||||
|
||||
return id, err
|
||||
|
||||
// ret, err := c.client.SubmitPerformerDraft(ctx, draft, uploadImage(image))
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
// id := ret.SubmitPerformerDraft.ID
|
||||
// return id, nil
|
||||
}
|
||||
@@ -1,589 +0,0 @@
|
||||
package stashbox
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/stashapp/stash/pkg/logger"
|
||||
"github.com/stashapp/stash/pkg/match"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/scraper"
|
||||
"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
|
||||
"github.com/stashapp/stash/pkg/sliceutil"
|
||||
"github.com/stashapp/stash/pkg/sliceutil/stringslice"
|
||||
"github.com/stashapp/stash/pkg/utils"
|
||||
)
|
||||
|
||||
// QueryStashBoxScene queries stash-box for scenes using a query string.
|
||||
func (c Client) QueryStashBoxScene(ctx context.Context, queryStr string) ([]*scraper.ScrapedScene, error) {
|
||||
scenes, err := c.client.SearchScene(ctx, queryStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sceneFragments := scenes.SearchScene
|
||||
|
||||
var ret []*scraper.ScrapedScene
|
||||
var ignoredTags []string
|
||||
for _, s := range sceneFragments {
|
||||
ss, err := c.sceneFragmentToScrapedScene(ctx, s)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var thisIgnoredTags []string
|
||||
ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
|
||||
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
|
||||
|
||||
ret = append(ret, ss)
|
||||
}
|
||||
|
||||
scraper.LogIgnoredTags(ignoredTags)
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
// FindStashBoxScenesByFingerprints queries stash-box for a scene using the
|
||||
// scene's MD5/OSHASH checksum, or PHash.
|
||||
func (c Client) FindStashBoxSceneByFingerprints(ctx context.Context, sceneID int) ([]*scraper.ScrapedScene, error) {
|
||||
res, err := c.FindStashBoxScenesByFingerprints(ctx, []int{sceneID})
|
||||
if len(res) > 0 {
|
||||
return res[0], err
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// FindStashBoxScenesByFingerprints queries stash-box for scenes using every
|
||||
// scene's MD5/OSHASH checksum, or PHash, and returns results in the same order
|
||||
// as the input slice.
|
||||
func (c Client) FindStashBoxScenesByFingerprints(ctx context.Context, ids []int) ([][]*scraper.ScrapedScene, error) {
|
||||
var fingerprints [][]*graphql.FingerprintQueryInput
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
qb := r.Scene
|
||||
|
||||
for _, sceneID := range ids {
|
||||
scene, err := qb.Find(ctx, sceneID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if scene == nil {
|
||||
return fmt.Errorf("scene with id %d not found", sceneID)
|
||||
}
|
||||
|
||||
if err := scene.LoadFiles(ctx, r.Scene); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var sceneFPs []*graphql.FingerprintQueryInput
|
||||
|
||||
for _, f := range scene.Files.List() {
|
||||
checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5)
|
||||
if checksum != "" {
|
||||
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
|
||||
Hash: checksum,
|
||||
Algorithm: graphql.FingerprintAlgorithmMd5,
|
||||
})
|
||||
}
|
||||
|
||||
oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash)
|
||||
if oshash != "" {
|
||||
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
|
||||
Hash: oshash,
|
||||
Algorithm: graphql.FingerprintAlgorithmOshash,
|
||||
})
|
||||
}
|
||||
|
||||
phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash)
|
||||
if phash != 0 {
|
||||
phashStr := utils.PhashToString(phash)
|
||||
sceneFPs = append(sceneFPs, &graphql.FingerprintQueryInput{
|
||||
Hash: phashStr,
|
||||
Algorithm: graphql.FingerprintAlgorithmPhash,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fingerprints = append(fingerprints, sceneFPs)
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return c.findStashBoxScenesByFingerprints(ctx, fingerprints)
|
||||
}
|
||||
|
||||
func (c Client) findStashBoxScenesByFingerprints(ctx context.Context, scenes [][]*graphql.FingerprintQueryInput) ([][]*scraper.ScrapedScene, error) {
|
||||
var results [][]*scraper.ScrapedScene
|
||||
|
||||
// filter out nils
|
||||
var validScenes [][]*graphql.FingerprintQueryInput
|
||||
for _, s := range scenes {
|
||||
if len(s) > 0 {
|
||||
validScenes = append(validScenes, s)
|
||||
}
|
||||
}
|
||||
|
||||
var ignoredTags []string
|
||||
|
||||
for i := 0; i < len(validScenes); i += 40 {
|
||||
end := i + 40
|
||||
if end > len(validScenes) {
|
||||
end = len(validScenes)
|
||||
}
|
||||
scenes, err := c.client.FindScenesBySceneFingerprints(ctx, validScenes[i:end])
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, sceneFragments := range scenes.FindScenesBySceneFingerprints {
|
||||
var sceneResults []*scraper.ScrapedScene
|
||||
for _, scene := range sceneFragments {
|
||||
ss, err := c.sceneFragmentToScrapedScene(ctx, scene)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var thisIgnoredTags []string
|
||||
ss.Tags, thisIgnoredTags = scraper.FilterTags(c.excludeTagRE, ss.Tags)
|
||||
ignoredTags = sliceutil.AppendUniques(ignoredTags, thisIgnoredTags)
|
||||
|
||||
sceneResults = append(sceneResults, ss)
|
||||
}
|
||||
results = append(results, sceneResults)
|
||||
}
|
||||
}
|
||||
|
||||
scraper.LogIgnoredTags(ignoredTags)
|
||||
|
||||
// repopulate the results to be the same order as the input
|
||||
ret := make([][]*scraper.ScrapedScene, len(scenes))
|
||||
upTo := 0
|
||||
|
||||
for i, v := range scenes {
|
||||
if len(v) > 0 {
|
||||
ret[i] = results[upTo]
|
||||
upTo++
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func (c Client) sceneFragmentToScrapedScene(ctx context.Context, s *graphql.SceneFragment) (*scraper.ScrapedScene, error) {
|
||||
stashID := s.ID
|
||||
|
||||
ss := &scraper.ScrapedScene{
|
||||
Title: s.Title,
|
||||
Code: s.Code,
|
||||
Date: s.Date,
|
||||
Details: s.Details,
|
||||
Director: s.Director,
|
||||
URL: findURL(s.Urls, "STUDIO"),
|
||||
Duration: s.Duration,
|
||||
RemoteSiteID: &stashID,
|
||||
Fingerprints: getFingerprints(s),
|
||||
// Image
|
||||
// stash_id
|
||||
}
|
||||
|
||||
for _, u := range s.Urls {
|
||||
ss.URLs = append(ss.URLs, u.URL)
|
||||
}
|
||||
|
||||
if len(ss.URLs) > 0 {
|
||||
ss.URL = &ss.URLs[0]
|
||||
}
|
||||
|
||||
if len(s.Images) > 0 {
|
||||
// TODO - #454 code sorts images by aspect ratio according to a wanted
|
||||
// orientation. I'm just grabbing the first for now
|
||||
ss.Image = getFirstImage(ctx, c.getHTTPClient(), s.Images)
|
||||
}
|
||||
|
||||
if ss.URL == nil && len(s.Urls) > 0 {
|
||||
// The scene in Stash-box may not have a Studio URL but it does have another URL.
|
||||
// For example it has a www.manyvids.com URL, which is auto set as type ManyVids.
|
||||
// This should be re-visited once Stashapp can support more than one URL.
|
||||
ss.URL = &s.Urls[0].URL
|
||||
}
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
pqb := r.Performer
|
||||
tqb := r.Tag
|
||||
|
||||
if s.Studio != nil {
|
||||
ss.Studio = studioFragmentToScrapedStudio(*s.Studio)
|
||||
|
||||
err := match.ScrapedStudio(ctx, r.Studio, ss.Studio, &c.box.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var parentStudio *graphql.FindStudio
|
||||
if s.Studio.Parent != nil {
|
||||
parentStudio, err = c.client.FindStudio(ctx, &s.Studio.Parent.ID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if parentStudio.FindStudio != nil {
|
||||
ss.Studio.Parent = studioFragmentToScrapedStudio(*parentStudio.FindStudio)
|
||||
|
||||
err = match.ScrapedStudio(ctx, r.Studio, ss.Studio.Parent, &c.box.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, p := range s.Performers {
|
||||
sp := performerFragmentToScrapedPerformer(*p.Performer)
|
||||
|
||||
err := match.ScrapedPerformer(ctx, pqb, sp, &c.box.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ss.Performers = append(ss.Performers, sp)
|
||||
}
|
||||
|
||||
for _, t := range s.Tags {
|
||||
st := &models.ScrapedTag{
|
||||
Name: t.Name,
|
||||
}
|
||||
|
||||
err := match.ScrapedTag(ctx, tqb, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ss.Tags = append(ss.Tags, st)
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return ss, nil
|
||||
}
|
||||
|
||||
func getFirstImage(ctx context.Context, client *http.Client, images []*graphql.ImageFragment) *string {
|
||||
ret, err := fetchImage(ctx, client, images[0].URL)
|
||||
if err != nil && !errors.Is(err, context.Canceled) {
|
||||
logger.Warnf("Error fetching image %s: %s", images[0].URL, err.Error())
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
func getFingerprints(scene *graphql.SceneFragment) []*models.StashBoxFingerprint {
|
||||
fingerprints := []*models.StashBoxFingerprint{}
|
||||
for _, fp := range scene.Fingerprints {
|
||||
fingerprint := models.StashBoxFingerprint{
|
||||
Algorithm: fp.Algorithm.String(),
|
||||
Hash: fp.Hash,
|
||||
Duration: fp.Duration,
|
||||
}
|
||||
fingerprints = append(fingerprints, &fingerprint)
|
||||
}
|
||||
return fingerprints
|
||||
}
|
||||
|
||||
func (c Client) SubmitSceneDraft(ctx context.Context, scene *models.Scene, cover []byte) (*string, error) {
|
||||
draft := graphql.SceneDraftInput{}
|
||||
var image io.Reader
|
||||
r := c.repository
|
||||
pqb := r.Performer
|
||||
sqb := r.Studio
|
||||
endpoint := c.box.Endpoint
|
||||
|
||||
if scene.Title != "" {
|
||||
draft.Title = &scene.Title
|
||||
}
|
||||
if scene.Code != "" {
|
||||
draft.Code = &scene.Code
|
||||
}
|
||||
if scene.Details != "" {
|
||||
draft.Details = &scene.Details
|
||||
}
|
||||
if scene.Director != "" {
|
||||
draft.Director = &scene.Director
|
||||
}
|
||||
// TODO - draft does not accept multiple URLs. Use single URL for now.
|
||||
if len(scene.URLs.List()) > 0 {
|
||||
url := strings.TrimSpace(scene.URLs.List()[0])
|
||||
draft.URL = &url
|
||||
}
|
||||
if scene.Date != nil {
|
||||
v := scene.Date.String()
|
||||
draft.Date = &v
|
||||
}
|
||||
|
||||
if scene.StudioID != nil {
|
||||
studio, err := sqb.Find(ctx, *scene.StudioID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if studio == nil {
|
||||
return nil, fmt.Errorf("studio with id %d not found", *scene.StudioID)
|
||||
}
|
||||
|
||||
studioDraft := graphql.DraftEntityInput{
|
||||
Name: studio.Name,
|
||||
}
|
||||
|
||||
stashIDs, err := sqb.GetStashIDs(ctx, studio.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, stashID := range stashIDs {
|
||||
c := stashID
|
||||
if stashID.Endpoint == endpoint {
|
||||
studioDraft.ID = &c.StashID
|
||||
break
|
||||
}
|
||||
}
|
||||
draft.Studio = &studioDraft
|
||||
}
|
||||
|
||||
fingerprints := []*graphql.FingerprintInput{}
|
||||
|
||||
// submit all file fingerprints
|
||||
if err := scene.LoadFiles(ctx, r.Scene); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, f := range scene.Files.List() {
|
||||
duration := f.Duration
|
||||
|
||||
if duration != 0 {
|
||||
if oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash); oshash != "" {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: oshash,
|
||||
Algorithm: graphql.FingerprintAlgorithmOshash,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
|
||||
}
|
||||
|
||||
if checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5); checksum != "" {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: checksum,
|
||||
Algorithm: graphql.FingerprintAlgorithmMd5,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
|
||||
}
|
||||
|
||||
if phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash); phash != 0 {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: utils.PhashToString(phash),
|
||||
Algorithm: graphql.FingerprintAlgorithmPhash,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = appendFingerprintUnique(fingerprints, &fingerprint)
|
||||
}
|
||||
}
|
||||
}
|
||||
draft.Fingerprints = fingerprints
|
||||
|
||||
scenePerformers, err := pqb.FindBySceneID(ctx, scene.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
performers := []*graphql.DraftEntityInput{}
|
||||
for _, p := range scenePerformers {
|
||||
performerDraft := graphql.DraftEntityInput{
|
||||
Name: p.Name,
|
||||
}
|
||||
|
||||
stashIDs, err := pqb.GetStashIDs(ctx, p.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for _, stashID := range stashIDs {
|
||||
c := stashID
|
||||
if stashID.Endpoint == endpoint {
|
||||
performerDraft.ID = &c.StashID
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
performers = append(performers, &performerDraft)
|
||||
}
|
||||
draft.Performers = performers
|
||||
|
||||
var tags []*graphql.DraftEntityInput
|
||||
sceneTags, err := r.Tag.FindBySceneID(ctx, scene.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, tag := range sceneTags {
|
||||
tags = append(tags, &graphql.DraftEntityInput{Name: tag.Name})
|
||||
}
|
||||
draft.Tags = tags
|
||||
|
||||
if len(cover) > 0 {
|
||||
image = bytes.NewReader(cover)
|
||||
}
|
||||
|
||||
if err := scene.LoadStashIDs(ctx, r.Scene); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
stashIDs := scene.StashIDs.List()
|
||||
var stashID *string
|
||||
for _, v := range stashIDs {
|
||||
if v.Endpoint == endpoint {
|
||||
vv := v.StashID
|
||||
stashID = &vv
|
||||
break
|
||||
}
|
||||
}
|
||||
draft.ID = stashID
|
||||
|
||||
var id *string
|
||||
var ret graphql.SubmitSceneDraft
|
||||
err = c.submitDraft(ctx, graphql.SubmitSceneDraftDocument, draft, image, &ret)
|
||||
id = ret.SubmitSceneDraft.ID
|
||||
|
||||
return id, err
|
||||
|
||||
// ret, err := c.client.SubmitSceneDraft(ctx, draft, uploadImage(image))
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
// id := ret.SubmitSceneDraft.ID
|
||||
// return id, nil
|
||||
}
|
||||
|
||||
func (c Client) SubmitStashBoxFingerprints(ctx context.Context, sceneIDs []string) (bool, error) {
|
||||
ids, err := stringslice.StringSliceToIntSlice(sceneIDs)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
endpoint := c.box.Endpoint
|
||||
|
||||
var fingerprints []graphql.FingerprintSubmission
|
||||
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
qb := r.Scene
|
||||
|
||||
for _, sceneID := range ids {
|
||||
scene, err := qb.Find(ctx, sceneID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if scene == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := scene.LoadStashIDs(ctx, qb); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := scene.LoadFiles(ctx, qb); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
stashIDs := scene.StashIDs.List()
|
||||
sceneStashID := ""
|
||||
for _, stashID := range stashIDs {
|
||||
if stashID.Endpoint == endpoint {
|
||||
sceneStashID = stashID.StashID
|
||||
}
|
||||
}
|
||||
|
||||
if sceneStashID != "" {
|
||||
for _, f := range scene.Files.List() {
|
||||
duration := f.Duration
|
||||
|
||||
if duration != 0 {
|
||||
if checksum := f.Fingerprints.GetString(models.FingerprintTypeMD5); checksum != "" {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: checksum,
|
||||
Algorithm: graphql.FingerprintAlgorithmMd5,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = append(fingerprints, graphql.FingerprintSubmission{
|
||||
SceneID: sceneStashID,
|
||||
Fingerprint: &fingerprint,
|
||||
})
|
||||
}
|
||||
|
||||
if oshash := f.Fingerprints.GetString(models.FingerprintTypeOshash); oshash != "" {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: oshash,
|
||||
Algorithm: graphql.FingerprintAlgorithmOshash,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = append(fingerprints, graphql.FingerprintSubmission{
|
||||
SceneID: sceneStashID,
|
||||
Fingerprint: &fingerprint,
|
||||
})
|
||||
}
|
||||
|
||||
if phash := f.Fingerprints.GetInt64(models.FingerprintTypePhash); phash != 0 {
|
||||
fingerprint := graphql.FingerprintInput{
|
||||
Hash: utils.PhashToString(phash),
|
||||
Algorithm: graphql.FingerprintAlgorithmPhash,
|
||||
Duration: int(duration),
|
||||
}
|
||||
fingerprints = append(fingerprints, graphql.FingerprintSubmission{
|
||||
SceneID: sceneStashID,
|
||||
Fingerprint: &fingerprint,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
return c.submitStashBoxFingerprints(ctx, fingerprints)
|
||||
}
|
||||
|
||||
func (c Client) submitStashBoxFingerprints(ctx context.Context, fingerprints []graphql.FingerprintSubmission) (bool, error) {
|
||||
for _, fingerprint := range fingerprints {
|
||||
_, err := c.client.SubmitFingerprint(ctx, fingerprint)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func appendFingerprintUnique(v []*graphql.FingerprintInput, toAdd *graphql.FingerprintInput) []*graphql.FingerprintInput {
|
||||
for _, vv := range v {
|
||||
if vv.Algorithm == toAdd.Algorithm && vv.Hash == toAdd.Hash {
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
return append(v, toAdd)
|
||||
}
|
||||
@@ -1,81 +0,0 @@
|
||||
package stashbox
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/stashapp/stash/pkg/match"
|
||||
"github.com/stashapp/stash/pkg/models"
|
||||
"github.com/stashapp/stash/pkg/scraper/stashbox/graphql"
|
||||
)
|
||||
|
||||
func (c Client) FindStashBoxStudio(ctx context.Context, query string) (*models.ScrapedStudio, error) {
|
||||
var studio *graphql.FindStudio
|
||||
|
||||
_, err := uuid.Parse(query)
|
||||
if err == nil {
|
||||
// Confirmed the user passed in a Stash ID
|
||||
studio, err = c.client.FindStudio(ctx, &query, nil)
|
||||
} else {
|
||||
// Otherwise assume they're searching on a name
|
||||
studio, err = c.client.FindStudio(ctx, nil, &query)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var ret *models.ScrapedStudio
|
||||
if studio.FindStudio != nil {
|
||||
r := c.repository
|
||||
if err := r.WithReadTxn(ctx, func(ctx context.Context) error {
|
||||
ret = studioFragmentToScrapedStudio(*studio.FindStudio)
|
||||
|
||||
err = match.ScrapedStudio(ctx, r.Studio, ret, &c.box.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if studio.FindStudio.Parent != nil {
|
||||
parentStudio, err := c.client.FindStudio(ctx, &studio.FindStudio.Parent.ID, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if parentStudio.FindStudio != nil {
|
||||
ret.Parent = studioFragmentToScrapedStudio(*parentStudio.FindStudio)
|
||||
|
||||
err = match.ScrapedStudio(ctx, r.Studio, ret.Parent, &c.box.Endpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func studioFragmentToScrapedStudio(s graphql.StudioFragment) *models.ScrapedStudio {
|
||||
images := []string{}
|
||||
for _, image := range s.Images {
|
||||
images = append(images, image.URL)
|
||||
}
|
||||
|
||||
st := &models.ScrapedStudio{
|
||||
Name: s.Name,
|
||||
URL: findURL(s.Urls, "HOME"),
|
||||
Images: images,
|
||||
RemoteSiteID: &s.ID,
|
||||
}
|
||||
|
||||
if len(st.Images) > 0 {
|
||||
st.Image = &st.Images[0]
|
||||
}
|
||||
|
||||
return st
|
||||
}
|
||||
@@ -151,7 +151,7 @@ func (s *xpathScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
|
||||
return nil, ErrNotSupported
|
||||
}
|
||||
|
||||
func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*ScrapedScene, error) {
|
||||
func (s *xpathScraper) scrapeSceneByScene(ctx context.Context, scene *models.Scene) (*models.ScrapedScene, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromScene(scene)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
@@ -210,7 +210,7 @@ func (s *xpathScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
|
||||
return scraper.scrapeScene(ctx, q)
|
||||
}
|
||||
|
||||
func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*ScrapedGallery, error) {
|
||||
func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *models.Gallery) (*models.ScrapedGallery, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromGallery(gallery)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
@@ -234,7 +234,7 @@ func (s *xpathScraper) scrapeGalleryByGallery(ctx context.Context, gallery *mode
|
||||
return scraper.scrapeGallery(ctx, q)
|
||||
}
|
||||
|
||||
func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*ScrapedImage, error) {
|
||||
func (s *xpathScraper) scrapeImageByImage(ctx context.Context, image *models.Image) (*models.ScrapedImage, error) {
|
||||
// construct the URL
|
||||
queryURL := queryURLParametersFromImage(image)
|
||||
if s.scraper.QueryURLReplacements != nil {
|
||||
|
||||
Reference in New Issue
Block a user