Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 12:54:38 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix scraping error (#704)

Browse Source
This commit is contained in:
WithoutPants
2020-08-04 20:43:56 +10:00
committed by GitHub
parent 4373f9bf01
commit b166abfa7b
1 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/scraper/url.go
View File

@@ -1,6 +1,7 @@
package scraper package scraper
import ( import (
"bytes"
"context" "context"
"errors" "errors"
"fmt" "fmt"
@@ -65,7 +66,14 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
} }
defer resp.Body.Close() defer resp.Body.Close()
return charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
bodyReader := bytes.NewReader(body)
return charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
} }
// func urlFromCDP uses chrome cdp and DOM to load and process the url // func urlFromCDP uses chrome cdp and DOM to load and process the url