mirror of
https://github.com/stashapp/stash.git
synced 2025-12-17 20:34:37 +03:00
Restructure scraping settings (#1548)
* Change scrapers overview into collapsible per type * Move scraping configuration to (renamed) scrapers tab Rename the Scrapers tab to Scraping and move the scraping configuration to this tab.
This commit is contained in:
@@ -64,6 +64,12 @@ fragment ConfigDLNAData on ConfigDLNAResult {
|
|||||||
interfaces
|
interfaces
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fragment ConfigScrapingData on ConfigScrapingResult {
|
||||||
|
scraperUserAgent
|
||||||
|
scraperCertCheck
|
||||||
|
scraperCDPPath
|
||||||
|
}
|
||||||
|
|
||||||
fragment ConfigData on ConfigResult {
|
fragment ConfigData on ConfigResult {
|
||||||
general {
|
general {
|
||||||
...ConfigGeneralData
|
...ConfigGeneralData
|
||||||
@@ -74,4 +80,7 @@ fragment ConfigData on ConfigResult {
|
|||||||
dlna {
|
dlna {
|
||||||
...ConfigDLNAData
|
...ConfigDLNAData
|
||||||
}
|
}
|
||||||
|
scraping {
|
||||||
|
...ConfigScrapingData
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,12 @@ mutation ConfigureDLNA($input: ConfigDLNAInput!) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mutation ConfigureScraping($input: ConfigScrapingInput!) {
|
||||||
|
configureScraping(input: $input) {
|
||||||
|
...ConfigScrapingData
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
mutation GenerateAPIKey($input: GenerateAPIKeyInput!) {
|
mutation GenerateAPIKey($input: GenerateAPIKeyInput!) {
|
||||||
generateAPIKey(input: $input)
|
generateAPIKey(input: $input)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -212,6 +212,7 @@ type Mutation {
|
|||||||
configureGeneral(input: ConfigGeneralInput!): ConfigGeneralResult!
|
configureGeneral(input: ConfigGeneralInput!): ConfigGeneralResult!
|
||||||
configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult!
|
configureInterface(input: ConfigInterfaceInput!): ConfigInterfaceResult!
|
||||||
configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult!
|
configureDLNA(input: ConfigDLNAInput!): ConfigDLNAResult!
|
||||||
|
configureScraping(input: ConfigScrapingInput!): ConfigScrapingResult!
|
||||||
|
|
||||||
"""Generate and set (or clear) API key"""
|
"""Generate and set (or clear) API key"""
|
||||||
generateAPIKey(input: GenerateAPIKeyInput!): String!
|
generateAPIKey(input: GenerateAPIKeyInput!): String!
|
||||||
|
|||||||
@@ -90,11 +90,11 @@ input ConfigGeneralInput {
|
|||||||
"""Array of file regexp to exclude from Image Scans"""
|
"""Array of file regexp to exclude from Image Scans"""
|
||||||
imageExcludes: [String!]
|
imageExcludes: [String!]
|
||||||
"""Scraper user agent string"""
|
"""Scraper user agent string"""
|
||||||
scraperUserAgent: String
|
scraperUserAgent: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
|
||||||
"""Scraper CDP path. Path to chrome executable or remote address"""
|
"""Scraper CDP path. Path to chrome executable or remote address"""
|
||||||
scraperCDPPath: String
|
scraperCDPPath: String @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
|
||||||
"""Whether the scraper should check for invalid certificates"""
|
"""Whether the scraper should check for invalid certificates"""
|
||||||
scraperCertCheck: Boolean!
|
scraperCertCheck: Boolean @deprecated(reason: "use mutation ConfigureScraping(input: ConfigScrapingInput) instead")
|
||||||
"""Stash-box instances used for tagging"""
|
"""Stash-box instances used for tagging"""
|
||||||
stashBoxes: [StashBoxInput!]!
|
stashBoxes: [StashBoxInput!]!
|
||||||
}
|
}
|
||||||
@@ -163,11 +163,11 @@ type ConfigGeneralResult {
|
|||||||
"""Array of file regexp to exclude from Image Scans"""
|
"""Array of file regexp to exclude from Image Scans"""
|
||||||
imageExcludes: [String!]!
|
imageExcludes: [String!]!
|
||||||
"""Scraper user agent string"""
|
"""Scraper user agent string"""
|
||||||
scraperUserAgent: String
|
scraperUserAgent: String @deprecated(reason: "use ConfigResult.scraping instead")
|
||||||
"""Scraper CDP path. Path to chrome executable or remote address"""
|
"""Scraper CDP path. Path to chrome executable or remote address"""
|
||||||
scraperCDPPath: String
|
scraperCDPPath: String @deprecated(reason: "use ConfigResult.scraping instead")
|
||||||
"""Whether the scraper should check for invalid certificates"""
|
"""Whether the scraper should check for invalid certificates"""
|
||||||
scraperCertCheck: Boolean!
|
scraperCertCheck: Boolean! @deprecated(reason: "use ConfigResult.scraping instead")
|
||||||
"""Stash-box instances used for tagging"""
|
"""Stash-box instances used for tagging"""
|
||||||
stashBoxes: [StashBox!]!
|
stashBoxes: [StashBox!]!
|
||||||
}
|
}
|
||||||
@@ -244,11 +244,30 @@ type ConfigDLNAResult {
|
|||||||
interfaces: [String!]!
|
interfaces: [String!]!
|
||||||
}
|
}
|
||||||
|
|
||||||
|
input ConfigScrapingInput {
|
||||||
|
"""Scraper user agent string"""
|
||||||
|
scraperUserAgent: String
|
||||||
|
"""Scraper CDP path. Path to chrome executable or remote address"""
|
||||||
|
scraperCDPPath: String
|
||||||
|
"""Whether the scraper should check for invalid certificates"""
|
||||||
|
scraperCertCheck: Boolean!
|
||||||
|
}
|
||||||
|
|
||||||
|
type ConfigScrapingResult {
|
||||||
|
"""Scraper user agent string"""
|
||||||
|
scraperUserAgent: String
|
||||||
|
"""Scraper CDP path. Path to chrome executable or remote address"""
|
||||||
|
scraperCDPPath: String
|
||||||
|
"""Whether the scraper should check for invalid certificates"""
|
||||||
|
scraperCertCheck: Boolean!
|
||||||
|
}
|
||||||
|
|
||||||
"""All configuration settings"""
|
"""All configuration settings"""
|
||||||
type ConfigResult {
|
type ConfigResult {
|
||||||
general: ConfigGeneralResult!
|
general: ConfigGeneralResult!
|
||||||
interface: ConfigInterfaceResult!
|
interface: ConfigInterfaceResult!
|
||||||
dlna: ConfigDLNAResult!
|
dlna: ConfigDLNAResult!
|
||||||
|
scraping: ConfigScrapingResult!
|
||||||
}
|
}
|
||||||
|
|
||||||
"""Directory structure of a path"""
|
"""Directory structure of a path"""
|
||||||
|
|||||||
@@ -178,7 +178,9 @@ func (r *mutationResolver) ConfigureGeneral(ctx context.Context, input models.Co
|
|||||||
refreshScraperCache = true
|
refreshScraperCache = true
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
|
if input.ScraperCertCheck != nil {
|
||||||
|
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
|
||||||
|
}
|
||||||
|
|
||||||
if input.StashBoxes != nil {
|
if input.StashBoxes != nil {
|
||||||
if err := c.ValidateStashBoxes(input.StashBoxes); err != nil {
|
if err := c.ValidateStashBoxes(input.StashBoxes); err != nil {
|
||||||
@@ -291,6 +293,31 @@ func (r *mutationResolver) ConfigureDlna(ctx context.Context, input models.Confi
|
|||||||
return makeConfigDLNAResult(), nil
|
return makeConfigDLNAResult(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *mutationResolver) ConfigureScraping(ctx context.Context, input models.ConfigScrapingInput) (*models.ConfigScrapingResult, error) {
|
||||||
|
c := config.GetInstance()
|
||||||
|
|
||||||
|
refreshScraperCache := false
|
||||||
|
if input.ScraperUserAgent != nil {
|
||||||
|
c.Set(config.ScraperUserAgent, input.ScraperUserAgent)
|
||||||
|
refreshScraperCache = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if input.ScraperCDPPath != nil {
|
||||||
|
c.Set(config.ScraperCDPPath, input.ScraperCDPPath)
|
||||||
|
refreshScraperCache = true
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Set(config.ScraperCertCheck, input.ScraperCertCheck)
|
||||||
|
if refreshScraperCache {
|
||||||
|
manager.GetInstance().RefreshScraperCache()
|
||||||
|
}
|
||||||
|
if err := c.Write(); err != nil {
|
||||||
|
return makeConfigScrapingResult(), err
|
||||||
|
}
|
||||||
|
|
||||||
|
return makeConfigScrapingResult(), nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) {
|
func (r *mutationResolver) GenerateAPIKey(ctx context.Context, input models.GenerateAPIKeyInput) (string, error) {
|
||||||
c := config.GetInstance()
|
c := config.GetInstance()
|
||||||
|
|
||||||
|
|||||||
@@ -39,6 +39,7 @@ func makeConfigResult() *models.ConfigResult {
|
|||||||
General: makeConfigGeneralResult(),
|
General: makeConfigGeneralResult(),
|
||||||
Interface: makeConfigInterfaceResult(),
|
Interface: makeConfigInterfaceResult(),
|
||||||
Dlna: makeConfigDLNAResult(),
|
Dlna: makeConfigDLNAResult(),
|
||||||
|
Scraping: makeConfigScrapingResult(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -132,3 +133,16 @@ func makeConfigDLNAResult() *models.ConfigDLNAResult {
|
|||||||
Interfaces: config.GetDLNAInterfaces(),
|
Interfaces: config.GetDLNAInterfaces(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func makeConfigScrapingResult() *models.ConfigScrapingResult {
|
||||||
|
config := config.GetInstance()
|
||||||
|
|
||||||
|
scraperUserAgent := config.GetScraperUserAgent()
|
||||||
|
scraperCDPPath := config.GetScraperCDPPath()
|
||||||
|
|
||||||
|
return &models.ConfigScrapingResult{
|
||||||
|
ScraperUserAgent: &scraperUserAgent,
|
||||||
|
ScraperCertCheck: config.GetScraperCertCheck(),
|
||||||
|
ScraperCDPPath: &scraperCDPPath,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
import React, { useEffect } from "react";
|
import React, { useEffect } from "react";
|
||||||
import { Route, Switch, useRouteMatch } from "react-router-dom";
|
import { Route, Switch, useRouteMatch } from "react-router-dom";
|
||||||
import { IntlProvider } from "react-intl";
|
import { IntlProvider } from "react-intl";
|
||||||
import { merge } from "lodash";
|
import { mergeWith } from "lodash";
|
||||||
import { ToastProvider } from "src/hooks/Toast";
|
import { ToastProvider } from "src/hooks/Toast";
|
||||||
import LightboxProvider from "src/hooks/Lightbox/context";
|
import LightboxProvider from "src/hooks/Lightbox/context";
|
||||||
import { library } from "@fortawesome/fontawesome-svg-core";
|
import { library } from "@fortawesome/fontawesome-svg-core";
|
||||||
@@ -59,11 +59,16 @@ export const App: React.FC = () => {
|
|||||||
const messageLanguage = languageMessageString(language);
|
const messageLanguage = languageMessageString(language);
|
||||||
|
|
||||||
// use en-GB as default messages if any messages aren't found in the chosen language
|
// use en-GB as default messages if any messages aren't found in the chosen language
|
||||||
const mergedMessages = merge(
|
const mergedMessages = mergeWith(
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(locales as any)[defaultMessageLanguage],
|
(locales as any)[defaultMessageLanguage],
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(locales as any)[messageLanguage]
|
(locales as any)[messageLanguage],
|
||||||
|
(objVal, srcVal) => {
|
||||||
|
if (srcVal === "") {
|
||||||
|
return objVal;
|
||||||
|
}
|
||||||
|
}
|
||||||
);
|
);
|
||||||
const messages = flattenMessages(mergedMessages);
|
const messages = flattenMessages(mergedMessages);
|
||||||
|
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
* Added not equals/greater than/less than modifiers for resolution criteria. ([#1568](https://github.com/stashapp/stash/pull/1568))
|
* Added not equals/greater than/less than modifiers for resolution criteria. ([#1568](https://github.com/stashapp/stash/pull/1568))
|
||||||
|
|
||||||
### 🎨 Improvements
|
### 🎨 Improvements
|
||||||
|
* Moved scraping settings into the Scraping settings page. ([#1548](https://github.com/stashapp/stash/pull/1548))
|
||||||
* Show current scene details in tagger view. ([#1605](https://github.com/stashapp/stash/pull/1605))
|
* Show current scene details in tagger view. ([#1605](https://github.com/stashapp/stash/pull/1605))
|
||||||
* Removed stripes and added background colour to default performer images (old images can be downloaded from the PR link). ([#1609](https://github.com/stashapp/stash/pull/1609))
|
* Removed stripes and added background colour to default performer images (old images can be downloaded from the PR link). ([#1609](https://github.com/stashapp/stash/pull/1609))
|
||||||
* Added pt-BR language option. ([#1587](https://github.com/stashapp/stash/pull/1587))
|
* Added pt-BR language option. ([#1587](https://github.com/stashapp/stash/pull/1587))
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import { SettingsInterfacePanel } from "./SettingsInterfacePanel/SettingsInterfa
|
|||||||
import { SettingsLogsPanel } from "./SettingsLogsPanel";
|
import { SettingsLogsPanel } from "./SettingsLogsPanel";
|
||||||
import { SettingsTasksPanel } from "./SettingsTasksPanel/SettingsTasksPanel";
|
import { SettingsTasksPanel } from "./SettingsTasksPanel/SettingsTasksPanel";
|
||||||
import { SettingsPluginsPanel } from "./SettingsPluginsPanel";
|
import { SettingsPluginsPanel } from "./SettingsPluginsPanel";
|
||||||
import { SettingsScrapersPanel } from "./SettingsScrapersPanel";
|
import { SettingsScrapingPanel } from "./SettingsScrapingPanel";
|
||||||
import { SettingsToolsPanel } from "./SettingsToolsPanel";
|
import { SettingsToolsPanel } from "./SettingsToolsPanel";
|
||||||
import { SettingsDLNAPanel } from "./SettingsDLNAPanel";
|
import { SettingsDLNAPanel } from "./SettingsDLNAPanel";
|
||||||
|
|
||||||
@@ -54,8 +54,8 @@ export const Settings: React.FC = () => {
|
|||||||
</Nav.Link>
|
</Nav.Link>
|
||||||
</Nav.Item>
|
</Nav.Item>
|
||||||
<Nav.Item>
|
<Nav.Item>
|
||||||
<Nav.Link eventKey="scrapers">
|
<Nav.Link eventKey="scraping">
|
||||||
<FormattedMessage id="config.categories.scrapers" />
|
<FormattedMessage id="config.categories.scraping" />
|
||||||
</Nav.Link>
|
</Nav.Link>
|
||||||
</Nav.Item>
|
</Nav.Item>
|
||||||
<Nav.Item>
|
<Nav.Item>
|
||||||
@@ -93,8 +93,8 @@ export const Settings: React.FC = () => {
|
|||||||
<Tab.Pane eventKey="tools" unmountOnExit>
|
<Tab.Pane eventKey="tools" unmountOnExit>
|
||||||
<SettingsToolsPanel />
|
<SettingsToolsPanel />
|
||||||
</Tab.Pane>
|
</Tab.Pane>
|
||||||
<Tab.Pane eventKey="scrapers" unmountOnExit>
|
<Tab.Pane eventKey="scraping" unmountOnExit>
|
||||||
<SettingsScrapersPanel />
|
<SettingsScrapingPanel />
|
||||||
</Tab.Pane>
|
</Tab.Pane>
|
||||||
<Tab.Pane eventKey="plugins" unmountOnExit>
|
<Tab.Pane eventKey="plugins" unmountOnExit>
|
||||||
<SettingsPluginsPanel />
|
<SettingsPluginsPanel />
|
||||||
|
|||||||
@@ -126,13 +126,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
|
|||||||
|
|
||||||
const [excludes, setExcludes] = useState<string[]>([]);
|
const [excludes, setExcludes] = useState<string[]>([]);
|
||||||
const [imageExcludes, setImageExcludes] = useState<string[]>([]);
|
const [imageExcludes, setImageExcludes] = useState<string[]>([]);
|
||||||
const [scraperUserAgent, setScraperUserAgent] = useState<string | undefined>(
|
|
||||||
undefined
|
|
||||||
);
|
|
||||||
const [scraperCDPPath, setScraperCDPPath] = useState<string | undefined>(
|
|
||||||
undefined
|
|
||||||
);
|
|
||||||
const [scraperCertCheck, setScraperCertCheck] = useState<boolean>(true);
|
|
||||||
const [stashBoxes, setStashBoxes] = useState<IStashBoxInstance[]>([]);
|
const [stashBoxes, setStashBoxes] = useState<IStashBoxInstance[]>([]);
|
||||||
|
|
||||||
const { data, error, loading } = useConfiguration();
|
const { data, error, loading } = useConfiguration();
|
||||||
@@ -173,9 +166,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
|
|||||||
galleryExtensions: commaDelimitedToList(galleryExtensions),
|
galleryExtensions: commaDelimitedToList(galleryExtensions),
|
||||||
excludes,
|
excludes,
|
||||||
imageExcludes,
|
imageExcludes,
|
||||||
scraperUserAgent,
|
|
||||||
scraperCDPPath,
|
|
||||||
scraperCertCheck,
|
|
||||||
stashBoxes: stashBoxes.map(
|
stashBoxes: stashBoxes.map(
|
||||||
(b) =>
|
(b) =>
|
||||||
({
|
({
|
||||||
@@ -223,9 +213,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
|
|||||||
);
|
);
|
||||||
setExcludes(conf.general.excludes);
|
setExcludes(conf.general.excludes);
|
||||||
setImageExcludes(conf.general.imageExcludes);
|
setImageExcludes(conf.general.imageExcludes);
|
||||||
setScraperUserAgent(conf.general.scraperUserAgent ?? undefined);
|
|
||||||
setScraperCDPPath(conf.general.scraperCDPPath ?? undefined);
|
|
||||||
setScraperCertCheck(conf.general.scraperCertCheck);
|
|
||||||
setStashBoxes(
|
setStashBoxes(
|
||||||
conf.general.stashBoxes.map((box, i) => ({
|
conf.general.stashBoxes.map((box, i) => ({
|
||||||
name: box?.name ?? undefined,
|
name: box?.name ?? undefined,
|
||||||
@@ -830,59 +817,6 @@ export const SettingsConfigurationPanel: React.FC = () => {
|
|||||||
</Form.Group>
|
</Form.Group>
|
||||||
</Form.Group>
|
</Form.Group>
|
||||||
|
|
||||||
<Form.Group>
|
|
||||||
<h4>{intl.formatMessage({ id: "config.general.scraping" })}</h4>
|
|
||||||
<Form.Group id="scraperUserAgent">
|
|
||||||
<h6>
|
|
||||||
{intl.formatMessage({ id: "config.general.scraper_user_agent" })}
|
|
||||||
</h6>
|
|
||||||
<Form.Control
|
|
||||||
className="col col-sm-6 text-input"
|
|
||||||
defaultValue={scraperUserAgent}
|
|
||||||
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
|
|
||||||
setScraperUserAgent(e.currentTarget.value)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Form.Text className="text-muted">
|
|
||||||
{intl.formatMessage({
|
|
||||||
id: "config.general.scraper_user_agent_desc",
|
|
||||||
})}
|
|
||||||
</Form.Text>
|
|
||||||
</Form.Group>
|
|
||||||
|
|
||||||
<Form.Group id="scraperCDPPath">
|
|
||||||
<h6>
|
|
||||||
{intl.formatMessage({ id: "config.general.chrome_cdp_path" })}
|
|
||||||
</h6>
|
|
||||||
<Form.Control
|
|
||||||
className="col col-sm-6 text-input"
|
|
||||||
defaultValue={scraperCDPPath}
|
|
||||||
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
|
|
||||||
setScraperCDPPath(e.currentTarget.value)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
<Form.Text className="text-muted">
|
|
||||||
{intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })}
|
|
||||||
</Form.Text>
|
|
||||||
</Form.Group>
|
|
||||||
|
|
||||||
<Form.Group>
|
|
||||||
<Form.Check
|
|
||||||
id="scaper-cert-check"
|
|
||||||
checked={scraperCertCheck}
|
|
||||||
label={intl.formatMessage({
|
|
||||||
id: "config.general.check_for_insecure_certificates",
|
|
||||||
})}
|
|
||||||
onChange={() => setScraperCertCheck(!scraperCertCheck)}
|
|
||||||
/>
|
|
||||||
<Form.Text className="text-muted">
|
|
||||||
{intl.formatMessage({
|
|
||||||
id: "config.general.check_for_insecure_certificates_desc",
|
|
||||||
})}
|
|
||||||
</Form.Text>
|
|
||||||
</Form.Group>
|
|
||||||
</Form.Group>
|
|
||||||
|
|
||||||
<hr />
|
<hr />
|
||||||
|
|
||||||
<Form.Group id="stashbox">
|
<Form.Group id="stashbox">
|
||||||
|
|||||||
@@ -1,16 +1,18 @@
|
|||||||
import React, { useState } from "react";
|
import React, { useEffect, useState } from "react";
|
||||||
import { FormattedMessage, useIntl } from "react-intl";
|
import { FormattedMessage, useIntl } from "react-intl";
|
||||||
import { Button } from "react-bootstrap";
|
import { Button, Form } from "react-bootstrap";
|
||||||
import {
|
import {
|
||||||
mutateReloadScrapers,
|
mutateReloadScrapers,
|
||||||
useListMovieScrapers,
|
useListMovieScrapers,
|
||||||
useListPerformerScrapers,
|
useListPerformerScrapers,
|
||||||
useListSceneScrapers,
|
useListSceneScrapers,
|
||||||
useListGalleryScrapers,
|
useListGalleryScrapers,
|
||||||
|
useConfiguration,
|
||||||
|
useConfigureScraping,
|
||||||
} from "src/core/StashService";
|
} from "src/core/StashService";
|
||||||
import { useToast } from "src/hooks";
|
import { useToast } from "src/hooks";
|
||||||
import { TextUtils } from "src/utils";
|
import { TextUtils } from "src/utils";
|
||||||
import { Icon, LoadingIndicator } from "src/components/Shared";
|
import { CollapseButton, Icon, LoadingIndicator } from "src/components/Shared";
|
||||||
import { ScrapeType } from "src/core/generated-graphql";
|
import { ScrapeType } from "src/core/generated-graphql";
|
||||||
|
|
||||||
interface IURLList {
|
interface IURLList {
|
||||||
@@ -67,7 +69,7 @@ const URLList: React.FC<IURLList> = ({ urls }) => {
|
|||||||
return <ul>{getListItems()}</ul>;
|
return <ul>{getListItems()}</ul>;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const SettingsScrapersPanel: React.FC = () => {
|
export const SettingsScrapingPanel: React.FC = () => {
|
||||||
const Toast = useToast();
|
const Toast = useToast();
|
||||||
const intl = useIntl();
|
const intl = useIntl();
|
||||||
const {
|
const {
|
||||||
@@ -87,17 +89,62 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
loading: loadingMovies,
|
loading: loadingMovies,
|
||||||
} = useListMovieScrapers();
|
} = useListMovieScrapers();
|
||||||
|
|
||||||
|
const [scraperUserAgent, setScraperUserAgent] = useState<string | undefined>(
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
const [scraperCDPPath, setScraperCDPPath] = useState<string | undefined>(
|
||||||
|
undefined
|
||||||
|
);
|
||||||
|
const [scraperCertCheck, setScraperCertCheck] = useState<boolean>(true);
|
||||||
|
|
||||||
|
const { data, error } = useConfiguration();
|
||||||
|
|
||||||
|
const [updateScrapingConfig] = useConfigureScraping({
|
||||||
|
scraperUserAgent,
|
||||||
|
scraperCDPPath,
|
||||||
|
scraperCertCheck,
|
||||||
|
});
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!data?.configuration || error) return;
|
||||||
|
|
||||||
|
const conf = data.configuration;
|
||||||
|
if (conf.scraping) {
|
||||||
|
setScraperUserAgent(conf.scraping.scraperUserAgent ?? undefined);
|
||||||
|
setScraperCDPPath(conf.scraping.scraperCDPPath ?? undefined);
|
||||||
|
setScraperCertCheck(conf.scraping.scraperCertCheck);
|
||||||
|
}
|
||||||
|
}, [data, error]);
|
||||||
|
|
||||||
async function onReloadScrapers() {
|
async function onReloadScrapers() {
|
||||||
await mutateReloadScrapers().catch((e) => Toast.error(e));
|
await mutateReloadScrapers().catch((e) => Toast.error(e));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function onSave() {
|
||||||
|
try {
|
||||||
|
await updateScrapingConfig();
|
||||||
|
Toast.success({
|
||||||
|
content: intl.formatMessage(
|
||||||
|
{ id: "toast.updated_entity" },
|
||||||
|
{
|
||||||
|
entity: intl
|
||||||
|
.formatMessage({ id: "configuration" })
|
||||||
|
.toLocaleLowerCase(),
|
||||||
|
}
|
||||||
|
),
|
||||||
|
});
|
||||||
|
} catch (e) {
|
||||||
|
Toast.error(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function renderPerformerScrapeTypes(types: ScrapeType[]) {
|
function renderPerformerScrapeTypes(types: ScrapeType[]) {
|
||||||
const typeStrings = types
|
const typeStrings = types
|
||||||
.filter((t) => t !== ScrapeType.Fragment)
|
.filter((t) => t !== ScrapeType.Fragment)
|
||||||
.map((t) => {
|
.map((t) => {
|
||||||
switch (t) {
|
switch (t) {
|
||||||
case ScrapeType.Name:
|
case ScrapeType.Name:
|
||||||
return intl.formatMessage({ id: "config.scrapers.search_by_name" });
|
return intl.formatMessage({ id: "config.scraping.search_by_name" });
|
||||||
default:
|
default:
|
||||||
return t;
|
return t;
|
||||||
}
|
}
|
||||||
@@ -117,7 +164,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
switch (t) {
|
switch (t) {
|
||||||
case ScrapeType.Fragment:
|
case ScrapeType.Fragment:
|
||||||
return intl.formatMessage(
|
return intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_metadata" },
|
{ id: "config.scraping.entity_metadata" },
|
||||||
{ entityType: intl.formatMessage({ id: "scene" }) }
|
{ entityType: intl.formatMessage({ id: "scene" }) }
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
@@ -139,7 +186,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
switch (t) {
|
switch (t) {
|
||||||
case ScrapeType.Fragment:
|
case ScrapeType.Fragment:
|
||||||
return intl.formatMessage(
|
return intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_metadata" },
|
{ id: "config.scraping.entity_metadata" },
|
||||||
{ entityType: intl.formatMessage({ id: "gallery" }) }
|
{ entityType: intl.formatMessage({ id: "gallery" }) }
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
@@ -161,7 +208,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
switch (t) {
|
switch (t) {
|
||||||
case ScrapeType.Fragment:
|
case ScrapeType.Fragment:
|
||||||
return intl.formatMessage(
|
return intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_metadata" },
|
{ id: "config.scraping.entity_metadata" },
|
||||||
{ entityType: intl.formatMessage({ id: "movie" }) }
|
{ entityType: intl.formatMessage({ id: "movie" }) }
|
||||||
);
|
);
|
||||||
default:
|
default:
|
||||||
@@ -195,7 +242,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
|
|
||||||
return renderTable(
|
return renderTable(
|
||||||
intl.formatMessage(
|
intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_scrapers" },
|
{ id: "config.scraping.entity_scrapers" },
|
||||||
{ entityType: intl.formatMessage({ id: "scene" }) }
|
{ entityType: intl.formatMessage({ id: "scene" }) }
|
||||||
),
|
),
|
||||||
elements
|
elements
|
||||||
@@ -217,7 +264,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
|
|
||||||
return renderTable(
|
return renderTable(
|
||||||
intl.formatMessage(
|
intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_scrapers" },
|
{ id: "config.scraping.entity_scrapers" },
|
||||||
{ entityType: intl.formatMessage({ id: "gallery" }) }
|
{ entityType: intl.formatMessage({ id: "gallery" }) }
|
||||||
),
|
),
|
||||||
elements
|
elements
|
||||||
@@ -241,7 +288,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
|
|
||||||
return renderTable(
|
return renderTable(
|
||||||
intl.formatMessage(
|
intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_scrapers" },
|
{ id: "config.scraping.entity_scrapers" },
|
||||||
{ entityType: intl.formatMessage({ id: "performer" }) }
|
{ entityType: intl.formatMessage({ id: "performer" }) }
|
||||||
),
|
),
|
||||||
elements
|
elements
|
||||||
@@ -261,7 +308,7 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
|
|
||||||
return renderTable(
|
return renderTable(
|
||||||
intl.formatMessage(
|
intl.formatMessage(
|
||||||
{ id: "config.scrapers.entity_scrapers" },
|
{ id: "config.scraping.entity_scrapers" },
|
||||||
{ entityType: intl.formatMessage({ id: "movie" }) }
|
{ entityType: intl.formatMessage({ id: "movie" }) }
|
||||||
),
|
),
|
||||||
elements
|
elements
|
||||||
@@ -271,25 +318,24 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
function renderTable(title: string, elements: JSX.Element[]) {
|
function renderTable(title: string, elements: JSX.Element[]) {
|
||||||
if (elements.length > 0) {
|
if (elements.length > 0) {
|
||||||
return (
|
return (
|
||||||
<div className="mb-2">
|
<CollapseButton text={title}>
|
||||||
<h5>{title}</h5>
|
|
||||||
<table className="scraper-table">
|
<table className="scraper-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>{intl.formatMessage({ id: "name" })}</th>
|
<th>{intl.formatMessage({ id: "name" })}</th>
|
||||||
<th>
|
<th>
|
||||||
{intl.formatMessage({
|
{intl.formatMessage({
|
||||||
id: "config.scrapers.supported_types",
|
id: "config.scraping.supported_types",
|
||||||
})}
|
})}
|
||||||
</th>
|
</th>
|
||||||
<th>
|
<th>
|
||||||
{intl.formatMessage({ id: "config.scrapers.supported_urls" })}
|
{intl.formatMessage({ id: "config.scraping.supported_urls" })}
|
||||||
</th>
|
</th>
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>{elements}</tbody>
|
<tbody>{elements}</tbody>
|
||||||
</table>
|
</table>
|
||||||
</div>
|
</CollapseButton>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -299,7 +345,63 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<>
|
<>
|
||||||
<h4>{intl.formatMessage({ id: "config.categories.scrapers" })}</h4>
|
<Form.Group>
|
||||||
|
<h4>{intl.formatMessage({ id: "config.general.scraping" })}</h4>
|
||||||
|
<Form.Group id="scraperUserAgent">
|
||||||
|
<h6>
|
||||||
|
{intl.formatMessage({ id: "config.general.scraper_user_agent" })}
|
||||||
|
</h6>
|
||||||
|
<Form.Control
|
||||||
|
className="col col-sm-6 text-input"
|
||||||
|
defaultValue={scraperUserAgent}
|
||||||
|
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
|
||||||
|
setScraperUserAgent(e.currentTarget.value)
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Form.Text className="text-muted">
|
||||||
|
{intl.formatMessage({
|
||||||
|
id: "config.general.scraper_user_agent_desc",
|
||||||
|
})}
|
||||||
|
</Form.Text>
|
||||||
|
</Form.Group>
|
||||||
|
|
||||||
|
<Form.Group id="scraperCDPPath">
|
||||||
|
<h6>
|
||||||
|
{intl.formatMessage({ id: "config.general.chrome_cdp_path" })}
|
||||||
|
</h6>
|
||||||
|
<Form.Control
|
||||||
|
className="col col-sm-6 text-input"
|
||||||
|
defaultValue={scraperCDPPath}
|
||||||
|
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
|
||||||
|
setScraperCDPPath(e.currentTarget.value)
|
||||||
|
}
|
||||||
|
/>
|
||||||
|
<Form.Text className="text-muted">
|
||||||
|
{intl.formatMessage({ id: "config.general.chrome_cdp_path_desc" })}
|
||||||
|
</Form.Text>
|
||||||
|
</Form.Group>
|
||||||
|
|
||||||
|
<Form.Group>
|
||||||
|
<Form.Check
|
||||||
|
id="scaper-cert-check"
|
||||||
|
checked={scraperCertCheck}
|
||||||
|
label={intl.formatMessage({
|
||||||
|
id: "config.general.check_for_insecure_certificates",
|
||||||
|
})}
|
||||||
|
onChange={() => setScraperCertCheck(!scraperCertCheck)}
|
||||||
|
/>
|
||||||
|
<Form.Text className="text-muted">
|
||||||
|
{intl.formatMessage({
|
||||||
|
id: "config.general.check_for_insecure_certificates_desc",
|
||||||
|
})}
|
||||||
|
</Form.Text>
|
||||||
|
</Form.Group>
|
||||||
|
</Form.Group>
|
||||||
|
|
||||||
|
<hr />
|
||||||
|
|
||||||
|
<h4>{intl.formatMessage({ id: "config.scraping.scrapers" })}</h4>
|
||||||
|
|
||||||
<div className="mb-3">
|
<div className="mb-3">
|
||||||
<Button onClick={() => onReloadScrapers()}>
|
<Button onClick={() => onReloadScrapers()}>
|
||||||
<span className="fa-icon">
|
<span className="fa-icon">
|
||||||
@@ -317,6 +419,12 @@ export const SettingsScrapersPanel: React.FC = () => {
|
|||||||
{renderPerformerScrapers()}
|
{renderPerformerScrapers()}
|
||||||
{renderMovieScrapers()}
|
{renderMovieScrapers()}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<hr />
|
||||||
|
|
||||||
|
<Button variant="primary" onClick={() => onSave()}>
|
||||||
|
<FormattedMessage id="actions.save" />
|
||||||
|
</Button>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
@@ -761,6 +761,13 @@ export const useRemoveTempDLNAIP = () => GQL.useRemoveTempDlnaipMutation();
|
|||||||
|
|
||||||
export const useLoggingSubscribe = () => GQL.useLoggingSubscribeSubscription();
|
export const useLoggingSubscribe = () => GQL.useLoggingSubscribeSubscription();
|
||||||
|
|
||||||
|
export const useConfigureScraping = (input: GQL.ConfigScrapingInput) =>
|
||||||
|
GQL.useConfigureScrapingMutation({
|
||||||
|
variables: { input },
|
||||||
|
refetchQueries: getQueryNames([GQL.ConfigurationDocument]),
|
||||||
|
update: deleteCache([GQL.ConfigurationDocument]),
|
||||||
|
});
|
||||||
|
|
||||||
export const querySystemStatus = () =>
|
export const querySystemStatus = () =>
|
||||||
client.query<GQL.SystemStatusQuery>({
|
client.query<GQL.SystemStatusQuery>({
|
||||||
query: GQL.SystemStatusDocument,
|
query: GQL.SystemStatusDocument,
|
||||||
|
|||||||
@@ -151,7 +151,7 @@
|
|||||||
"interface": "Oberfläche",
|
"interface": "Oberfläche",
|
||||||
"logs": "Protokoll",
|
"logs": "Protokoll",
|
||||||
"plugins": "Plugins",
|
"plugins": "Plugins",
|
||||||
"scrapers": "Scraper",
|
"scraping": "Scraping",
|
||||||
"tasks": "Aufgaben",
|
"tasks": "Aufgaben",
|
||||||
"tools": "Werkzeuge"
|
"tools": "Werkzeuge"
|
||||||
},
|
},
|
||||||
@@ -240,9 +240,10 @@
|
|||||||
"hooks": "Hooks",
|
"hooks": "Hooks",
|
||||||
"triggers_on": "Auslösen bei"
|
"triggers_on": "Auslösen bei"
|
||||||
},
|
},
|
||||||
"scrapers": {
|
"scraping": {
|
||||||
"entity_metadata": "{entityType} Metadaten",
|
"entity_metadata": "{entityType} Metadaten",
|
||||||
"entity_scrapers": "{entityType} Scraper",
|
"entity_scrapers": "{entityType} Scraper",
|
||||||
|
"scrapers": "Scraper",
|
||||||
"search_by_name": "Suche nach Name",
|
"search_by_name": "Suche nach Name",
|
||||||
"supported_types": "Unterstützte Typen",
|
"supported_types": "Unterstützte Typen",
|
||||||
"supported_urls": "URLs"
|
"supported_urls": "URLs"
|
||||||
|
|||||||
@@ -151,7 +151,7 @@
|
|||||||
"interface": "Interface",
|
"interface": "Interface",
|
||||||
"logs": "Logs",
|
"logs": "Logs",
|
||||||
"plugins": "Plugins",
|
"plugins": "Plugins",
|
||||||
"scrapers": "Scrapers",
|
"scraping": "Scraping",
|
||||||
"tasks": "Tasks",
|
"tasks": "Tasks",
|
||||||
"tools": "Tools"
|
"tools": "Tools"
|
||||||
},
|
},
|
||||||
@@ -240,9 +240,10 @@
|
|||||||
"hooks": "Hooks",
|
"hooks": "Hooks",
|
||||||
"triggers_on": "Triggers on"
|
"triggers_on": "Triggers on"
|
||||||
},
|
},
|
||||||
"scrapers": {
|
"scraping": {
|
||||||
"entity_metadata": "{entityType} Metadata",
|
"entity_metadata": "{entityType} Metadata",
|
||||||
"entity_scrapers": "{entityType} scrapers",
|
"entity_scrapers": "{entityType} scrapers",
|
||||||
|
"scrapers": "Scrapers",
|
||||||
"search_by_name": "Search by name",
|
"search_by_name": "Search by name",
|
||||||
"supported_types": "Supported types",
|
"supported_types": "Supported types",
|
||||||
"supported_urls": "URLs"
|
"supported_urls": "URLs"
|
||||||
|
|||||||
@@ -151,7 +151,7 @@
|
|||||||
"interface": "Interface",
|
"interface": "Interface",
|
||||||
"logs": "Logs",
|
"logs": "Logs",
|
||||||
"plugins": "Plugins",
|
"plugins": "Plugins",
|
||||||
"scrapers": "Scrapers",
|
"scraping": "Scraping",
|
||||||
"tasks": "Tarefas",
|
"tasks": "Tarefas",
|
||||||
"tools": "Ferramentas"
|
"tools": "Ferramentas"
|
||||||
},
|
},
|
||||||
@@ -240,9 +240,10 @@
|
|||||||
"hooks": "Hooks",
|
"hooks": "Hooks",
|
||||||
"triggers_on": "Triggers on"
|
"triggers_on": "Triggers on"
|
||||||
},
|
},
|
||||||
"scrapers": {
|
"scraping": {
|
||||||
"entity_metadata": "{entityType} metadados",
|
"entity_metadata": "{entityType} metadados",
|
||||||
"entity_scrapers": "{entityType} scrapers",
|
"entity_scrapers": "{entityType} scrapers",
|
||||||
|
"scrapers": "Scrapers",
|
||||||
"search_by_name": "Buscar por nome",
|
"search_by_name": "Buscar por nome",
|
||||||
"supported_types": "Tipos suportados",
|
"supported_types": "Tipos suportados",
|
||||||
"supported_urls": "URLs"
|
"supported_urls": "URLs"
|
||||||
|
|||||||
@@ -144,7 +144,7 @@
|
|||||||
"interface": "介面",
|
"interface": "介面",
|
||||||
"logs": "日誌",
|
"logs": "日誌",
|
||||||
"plugins": "插件",
|
"plugins": "插件",
|
||||||
"scrapers": "爬蟲",
|
"scraping": "爬蟲設定",
|
||||||
"tasks": "排程",
|
"tasks": "排程",
|
||||||
"tools": "工具"
|
"tools": "工具"
|
||||||
},
|
},
|
||||||
@@ -229,9 +229,10 @@
|
|||||||
"logs": {
|
"logs": {
|
||||||
"log_level": "日誌級別"
|
"log_level": "日誌級別"
|
||||||
},
|
},
|
||||||
"scrapers": {
|
"scraping": {
|
||||||
"entity_metadata": "{entityType}資訊",
|
"entity_metadata": "{entityType}資訊",
|
||||||
"entity_scrapers": "{entityType}爬蟲",
|
"entity_scrapers": "{entityType}爬蟲",
|
||||||
|
"scrapers": "爬蟲",
|
||||||
"search_by_name": "透過名稱搜尋",
|
"search_by_name": "透過名稱搜尋",
|
||||||
"supported_types": "支援類型",
|
"supported_types": "支援類型",
|
||||||
"supported_urls": "支援網址"
|
"supported_urls": "支援網址"
|
||||||
|
|||||||
Reference in New Issue
Block a user