Mirrors/stash
1
0
Fork 0
mirror of https://github.com/stashapp/stash.git synced 2025-12-18 04:44:37 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix scraping error (#704)

Browse Source
This commit is contained in:
WithoutPants
2020-08-04 20:43:56 +10:00
committed by GitHub
parent 4373f9bf01
commit b166abfa7b
1 changed files with 9 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
pkg/scraper/url.go
View File

@@ -1,6 +1,7 @@
package scraper
import (
"bytes"
"context"
"errors"
"fmt"
@@ -65,7 +66,14 @@ func loadURL(url string, scraperConfig config, globalConfig GlobalConfig) (io.Re
}
defer resp.Body.Close()
return charset.NewReader(resp.Body, resp.Header.Get("Content-Type"))
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
bodyReader := bytes.NewReader(body)
return charset.NewReader(bodyReader, resp.Header.Get("Content-Type"))
}
// func urlFromCDP uses chrome cdp and DOM to load and process the url