Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-16 20:07:05 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix stash scraper errors and add apikey field (#5474)

Browse Source 
* Use hasura/go-graphql-client instead of shurcooL version
* Fix graphql query errors
* Support setting api key for stash server
This commit is contained in:
WithoutPants
2024-11-13 10:14:55 +11:00
committed by GitHub
parent 64fed3553a
commit 5f690d96bd
9 changed files with 195 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/plugin/examples/common/graphql.go
View File

@@ -8,7 +8,7 @@ import (
"errors"
"fmt"
"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"
"github.com/stashapp/stash/pkg/plugin/common/log"
)

2
pkg/plugin/util/client.go
View File

@@ -8,7 +8,7 @@ import (
"net/url"
"strconv"
"github.com/shurcooL/graphql"
graphql "github.com/hasura/go-graphql-client"
"github.com/stashapp/stash/pkg/plugin/common"
)

3
pkg/scraper/config.go
View File

@@ -114,7 +114,8 @@ func (c config) validate() error {
}
type stashServer struct {
URL string `yaml:"url"`
URL string `yaml:"url"`
ApiKey string `yaml:"apiKey"`
}
type scraperTypeConfig struct {

55
pkg/scraper/graphql.go Normal file
View File

@@ -0,0 +1,55 @@
package scraper
import (
"errors"
"strings"
"github.com/hasura/go-graphql-client"
)
type graphqlErrors []error
func (e graphqlErrors) Error() string {
b := strings.Builder{}
for _, err := range e {
_, _ = b.WriteString(err.Error())
}
return b.String()
}
type graphqlError struct {
err graphql.Error
}
func (e graphqlError) Error() string {
unwrapped := e.err.Unwrap()
if unwrapped != nil {
var networkErr graphql.NetworkError
if errors.As(unwrapped, &networkErr) {
if networkErr.StatusCode() == 422 {
return networkErr.Body()
}
}
}
return e.err.Error()
}
// convertGraphqlError converts a graphql.Error or graphql.Errors into an error with a useful message.
// graphql.Error swallows important information, so we need to convert it to a more useful error type.
func convertGraphqlError(err error) error {
var gqlErrs graphql.Errors
if errors.As(err, &gqlErrs) {
ret := make(graphqlErrors, len(gqlErrs))
for i, e := range gqlErrs {
ret[i] = convertGraphqlError(e)
}
return ret
}
var gqlErr graphql.Error
if errors.As(err, &gqlErr) {
return graphqlError{gqlErr}
}
return err
}

33
pkg/scraper/image.go
View File

@@ -122,13 +122,19 @@ func setGroupBackImage(ctx context.Context, client *http.Client, m *models.Scrap
return nil
}
func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
type imageGetter struct {
client *http.Client
globalConfig GlobalConfig
requestModifier func(req *http.Request)
}
func (i *imageGetter) getImage(ctx context.Context, url string) (*string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
userAgent := globalConfig.GetScraperUserAgent()
userAgent := i.globalConfig.GetScraperUserAgent()
if userAgent != "" {
req.Header.Set("User-Agent", userAgent)
}
@@ -140,7 +146,11 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
req.Header.Set("Referer", req.URL.Scheme+"://"+req.Host+"/")
}
resp, err := client.Do(req)
if i.requestModifier != nil {
i.requestModifier(req)
}
resp, err := i.client.Do(req)
if err != nil {
return nil, err
@@ -167,10 +177,19 @@ func getImage(ctx context.Context, url string, client *http.Client, globalConfig
return &img, nil
}
func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/performer/"+performerID+"/image", client, globalConfig)
func getImage(ctx context.Context, url string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
g := imageGetter{
client: client,
globalConfig: globalConfig,
}
return g.getImage(ctx, url)
}
func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, client *http.Client, globalConfig GlobalConfig) (*string, error) {
return getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot", client, globalConfig)
func getStashPerformerImage(ctx context.Context, stashURL string, performerID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/performer/"+performerID+"/image")
}
func getStashSceneImage(ctx context.Context, stashURL string, sceneID string, imageGetter imageGetter) (*string, error) {
return imageGetter.getImage(ctx, stashURL+"/scene/"+sceneID+"/screenshot")
}

122
pkg/scraper/stash.go
View File

@@ -4,9 +4,11 @@ import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
graphql "github.com/hasura/go-graphql-client"
"github.com/jinzhu/copier"
"github.com/shurcooL/graphql"
"github.com/stashapp/stash/pkg/models"
)
@@ -27,9 +29,21 @@ func newStashScraper(scraper scraperTypeConfig, client *http.Client, config conf
}
}
func setApiKeyHeader(apiKey string) func(req *http.Request) {
return func(req *http.Request) {
req.Header.Set("ApiKey", apiKey)
}
}
func (s *stashScraper) getStashClient() *graphql.Client {
url := s.config.StashServer.URL
return graphql.NewClient(url+"/graphql", nil)
url := s.config.StashServer.URL + "/graphql"
ret := graphql.NewClient(url, s.client)
if s.config.StashServer.ApiKey != "" {
ret = ret.WithRequestModifier(setApiKeyHeader(s.config.StashServer.ApiKey))
}
return ret
}
type stashFindPerformerNamePerformer struct {
@@ -58,14 +72,12 @@ type scrapedTagStash struct {
type scrapedPerformerStash struct {
Name *string `graphql:"name" json:"name"`
Gender *string `graphql:"gender" json:"gender"`
URL *string `graphql:"url" json:"url"`
Twitter *string `graphql:"twitter" json:"twitter"`
Instagram *string `graphql:"instagram" json:"instagram"`
URLs []string `graphql:"urls" json:"urls"`
Birthdate *string `graphql:"birthdate" json:"birthdate"`
Ethnicity *string `graphql:"ethnicity" json:"ethnicity"`
Country *string `graphql:"country" json:"country"`
EyeColor *string `graphql:"eye_color" json:"eye_color"`
Height *string `graphql:"height" json:"height"`
Height *int `graphql:"height_cm" json:"height_cm"`
Measurements *string `graphql:"measurements" json:"measurements"`
FakeTits *string `graphql:"fake_tits" json:"fake_tits"`
PenisLength *string `graphql:"penis_length" json:"penis_length"`
@@ -73,12 +85,25 @@ type scrapedPerformerStash struct {
CareerLength *string `graphql:"career_length" json:"career_length"`
Tattoos *string `graphql:"tattoos" json:"tattoos"`
Piercings *string `graphql:"piercings" json:"piercings"`
Aliases *string `graphql:"aliases" json:"aliases"`
Aliases []string `graphql:"alias_list" json:"alias_list"`
Tags []*scrapedTagStash `graphql:"tags" json:"tags"`
Details *string `graphql:"details" json:"details"`
DeathDate *string `graphql:"death_date" json:"death_date"`
HairColor *string `graphql:"hair_color" json:"hair_color"`
Weight *string `graphql:"weight" json:"weight"`
Weight *int `graphql:"weight" json:"weight"`
}
func (s *stashScraper) imageGetter() imageGetter {
ret := imageGetter{
client: s.client,
globalConfig: s.globalConfig,
}
if s.config.StashServer.ApiKey != "" {
ret.requestModifier = setApiKeyHeader(s.config.StashServer.ApiKey)
}
return ret
}
func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (ScrapedContent, error) {
@@ -102,12 +127,12 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
// get the id from the URL field
vars := map[string]interface{}{
"f": performerID,
"f": graphql.ID(performerID),
}
err := client.Query(ctx, &q, vars)
if err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
// need to copy back to a scraped performer
@@ -117,11 +142,28 @@ func (s *stashScraper) scrapeByFragment(ctx context.Context, input Input) (Scrap
return nil, err
}
// convert alias list to aliases
aliasStr := strings.Join(q.FindPerformer.Aliases, ", ")
ret.Aliases = &aliasStr
// convert numeric to string
if q.FindPerformer.Height != nil {
heightStr := strconv.Itoa(*q.FindPerformer.Height)
ret.Height = &heightStr
}
if q.FindPerformer.Weight != nil {
weightStr := strconv.Itoa(*q.FindPerformer.Weight)
ret.Weight = &weightStr
}
// get the performer image directly
ret.Image, err = getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, s.client, s.globalConfig)
ig := s.imageGetter()
img, err := getStashPerformerImage(ctx, s.config.StashServer.URL, performerID, ig)
if err != nil {
return nil, err
}
ret.Images = []string{*img}
ret.Image = img
return &ret, nil
}
@@ -143,8 +185,15 @@ func (s *stashScraper) scrapedStashSceneToScrapedScene(ctx context.Context, scen
return nil, err
}
// get the performer image directly
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, s.client, s.globalConfig)
// convert first in files to file
if len(scene.Files) > 0 {
f := scene.Files[0].SceneFileType()
ret.File = &f
}
// get the scene image directly
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, scene.ID, ig)
if err != nil {
return nil, err
}
@@ -175,7 +224,7 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
err := client.Query(ctx, &q, vars)
if err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
for _, scene := range q.FindScenes.Scenes {
@@ -207,13 +256,41 @@ func (s *stashScraper) scrapeByName(ctx context.Context, name string, ty ScrapeC
return nil, ErrNotSupported
}
type stashVideoFile struct {
Size int64 `graphql:"size" json:"size"`
Duration float64 `graphql:"duration" json:"duration"`
VideoCodec string `graphql:"video_codec" json:"video_codec"`
AudioCodec string `graphql:"audio_codec" json:"audio_codec"`
Width int `graphql:"width" json:"width"`
Height int `graphql:"height" json:"height"`
Framerate float64 `graphql:"frame_rate" json:"frame_rate"`
Bitrate int `graphql:"bit_rate" json:"bit_rate"`
}
func (f stashVideoFile) SceneFileType() models.SceneFileType {
ret := models.SceneFileType{
Duration: &f.Duration,
VideoCodec: &f.VideoCodec,
AudioCodec: &f.AudioCodec,
Width: &f.Width,
Height: &f.Height,
Framerate: &f.Framerate,
Bitrate: &f.Bitrate,
}
size := strconv.FormatInt(f.Size, 10)
ret.Size = &size
return ret
}
type scrapedSceneStash struct {
ID string `graphql:"id" json:"id"`
Title *string `graphql:"title" json:"title"`
Details *string `graphql:"details" json:"details"`
URL *string `graphql:"url" json:"url"`
URLs []string `graphql:"urls" json:"urls"`
Date *string `graphql:"date" json:"date"`
File *models.SceneFileType `graphql:"file" json:"file"`
Files []stashVideoFile `graphql:"files" json:"files"`
Studio *scrapedStudioStash `graphql:"studio" json:"studio"`
Tags []*scrapedTagStash `graphql:"tags" json:"tags"`
Performers []*scrapedPerformerStash `graphql:"performers" json:"performers"`
@@ -239,12 +316,16 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
}
vars := map[string]interface{}{
"c": &input,
"c": input,
}
client := s.getStashClient()
if err := client.Query(ctx, &q, vars); err != nil {
return nil, err
return nil, convertGraphqlError(err)
}
if q.FindScene == nil {
return nil, nil
}
// need to copy back to a scraped scene
@@ -254,7 +335,8 @@ func (s *stashScraper) scrapeSceneByScene(ctx context.Context, scene *models.Sce
}
// get the performer image directly
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, s.client, s.globalConfig)
ig := s.imageGetter()
ret.Image, err = getStashSceneImage(ctx, s.config.StashServer.URL, q.FindScene.ID, ig)
if err != nil {
return nil, err
}