Scraper and plugin manager (#4242)

* Add package manager
* Add SettingModal validate
* Reverse modal button order
* Add plugin package management
* Refactor ClearableInput

ui/v2.5/src/components/List/Filters/SelectableFilter.tsx

@@ -102,6 +102,7 @@ const SelectableFilter: React.FC<ISelectableFilter> = ({
   onSelect,
   onUnselect,
 }) => {
+  const intl = useIntl();
   const objects = useMemo(() => {
     return queryResults.filter(
       (p) =>
@@ -124,6 +125,7 @@ const SelectableFilter: React.FC<ISelectableFilter> = ({
         focus={inputFocus}
         value={query}
         setValue={(v) => onQueryChange(v)}
+        placeholder={`${intl.formatMessage({ id: "actions.search" })}…`}
       />
       <ul>
         {selected.map((p) => (

ui/v2.5/src/components/Settings/Inputs.tsx

@@ -253,6 +253,7 @@ export interface ISettingModal<T> {
   close: (v?: T) => void;
   renderField: (value: T | undefined, setValue: (v?: T) => void) => JSX.Element;
   modalProps?: ModalProps;
+  validate?: (v: T) => boolean | undefined;
 }
 
 export const SettingModal = <T extends {}>(props: ISettingModal<T>) => {
@@ -265,6 +266,7 @@ export const SettingModal = <T extends {}>(props: ISettingModal<T>) => {
     close,
     renderField,
     modalProps,
+    validate,
   } = props;
 
   const intl = useIntl();
@@ -299,6 +301,7 @@ export const SettingModal = <T extends {}>(props: ISettingModal<T>) => {
             type="submit"
             variant="primary"
             onClick={() => close(currentValue)}
+            disabled={!currentValue || (validate && !validate(currentValue))}
           >
             <FormattedMessage id="actions.confirm" />
           </Button>

ui/v2.5/src/components/Settings/PluginPackageManager.tsx

@@ -0,0 +1,227 @@
+import React, { useEffect, useState, useMemo } from "react";
+import * as GQL from "src/core/generated-graphql";
+import {
+  evictQueries,
+  getClient,
+  queryAvailablePluginPackages,
+  useInstallPluginPackages,
+  useInstalledPluginPackages,
+  useInstalledPluginPackagesStatus,
+  useUninstallPluginPackages,
+  useUpdatePluginPackages,
+} from "src/core/StashService";
+import { useMonitorJob } from "src/utils/job";
+import {
+  AvailablePackages,
+  InstalledPackages,
+  RemotePackage,
+} from "../Shared/PackageManager/PackageManager";
+import { useSettings } from "./context";
+import { LoadingIndicator } from "../Shared/LoadingIndicator";
+import { SettingSection } from "./SettingSection";
+
+const impactedPackageChangeQueries = [
+  GQL.PluginsDocument,
+  GQL.PluginTasksDocument,
+  GQL.InstalledPluginPackagesDocument,
+  GQL.InstalledPluginPackagesStatusDocument,
+];
+
+export const InstalledPluginPackages: React.FC = () => {
+  const [loadUpgrades, setLoadUpgrades] = useState(false);
+  const [jobID, setJobID] = useState<string>();
+  const { job } = useMonitorJob(jobID, () => onPackageChanges());
+
+  const { data: installedPlugins, refetch: refetchPackages1 } =
+    useInstalledPluginPackages({
+      skip: loadUpgrades,
+    });
+
+  const {
+    data: withStatus,
+    refetch: refetchPackages2,
+    loading: statusLoading,
+  } = useInstalledPluginPackagesStatus({
+    skip: !loadUpgrades,
+  });
+
+  const [updatePackages] = useUpdatePluginPackages();
+  const [uninstallPackages] = useUninstallPluginPackages();
+
+  async function onUpdatePackages(packages: GQL.PackageSpecInput[]) {
+    const r = await updatePackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.updatePackages);
+  }
+
+  async function onUninstallPackages(packages: GQL.PackageSpecInput[]) {
+    const r = await uninstallPackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.uninstallPackages);
+  }
+
+  function refetchPackages() {
+    refetchPackages1();
+    refetchPackages2();
+  }
+
+  function onPackageChanges() {
+    // job is complete, refresh all local data
+    const ac = getClient();
+    evictQueries(ac.cache, impactedPackageChangeQueries);
+  }
+
+  function onCheckForUpdates() {
+    if (!loadUpgrades) {
+      setLoadUpgrades(true);
+    } else {
+      refetchPackages();
+    }
+  }
+
+  const installedPackages = useMemo(() => {
+    if (withStatus?.installedPackages) {
+      return withStatus.installedPackages;
+    }
+
+    return installedPlugins?.installedPackages ?? [];
+  }, [installedPlugins, withStatus]);
+
+  const loading = !!job || statusLoading;
+
+  return (
+    <SettingSection headingID="config.plugins.installed_plugins">
+      <div className="package-manager">
+        <InstalledPackages
+          loading={loading}
+          packages={installedPackages}
+          onCheckForUpdates={onCheckForUpdates}
+          onUpdatePackages={(packages) =>
+            onUpdatePackages(
+              packages.map((p) => ({
+                id: p.package_id,
+                sourceURL: p.upgrade!.sourceURL,
+              }))
+            )
+          }
+          onUninstallPackages={(packages) =>
+            onUninstallPackages(
+              packages.map((p) => ({
+                id: p.package_id,
+                sourceURL: p.sourceURL,
+              }))
+            )
+          }
+          updatesLoaded={loadUpgrades}
+        />
+      </div>
+    </SettingSection>
+  );
+};
+
+export const AvailablePluginPackages: React.FC = () => {
+  const { general, loading: configLoading, error, saveGeneral } = useSettings();
+
+  const [sources, setSources] = useState<GQL.PackageSource[]>();
+  const [jobID, setJobID] = useState<string>();
+  const { job } = useMonitorJob(jobID, () => onPackageChanges());
+
+  const [installPackages] = useInstallPluginPackages();
+
+  async function onInstallPackages(packages: GQL.PackageSpecInput[]) {
+    const r = await installPackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.installPackages);
+  }
+
+  function onPackageChanges() {
+    // job is complete, refresh all local data
+    const ac = getClient();
+    evictQueries(ac.cache, impactedPackageChangeQueries);
+  }
+
+  useEffect(() => {
+    if (!sources && !configLoading && general.pluginPackageSources) {
+      setSources(general.pluginPackageSources);
+    }
+  }, [sources, configLoading, general.pluginPackageSources]);
+
+  async function loadSource(source: string): Promise<RemotePackage[]> {
+    const { data } = await queryAvailablePluginPackages(source);
+    return data.availablePackages;
+  }
+
+  function addSource(source: GQL.PackageSource) {
+    saveGeneral({
+      pluginPackageSources: [...(general.pluginPackageSources ?? []), source],
+    });
+
+    setSources((prev) => {
+      return [...(prev ?? []), source];
+    });
+  }
+
+  function editSource(existing: GQL.PackageSource, changed: GQL.PackageSource) {
+    saveGeneral({
+      pluginPackageSources: general.pluginPackageSources?.map((s) =>
+        s.url === existing.url ? changed : s
+      ),
+    });
+
+    setSources((prev) => {
+      return prev?.map((s) => (s.url === existing.url ? changed : s));
+    });
+  }
+
+  function deleteSource(source: GQL.PackageSource) {
+    saveGeneral({
+      pluginPackageSources: general.pluginPackageSources?.filter(
+        (s) => s.url !== source.url
+      ),
+    });
+
+    setSources((prev) => {
+      return prev?.filter((s) => s.url !== source.url);
+    });
+  }
+
+  function renderDescription(pkg: RemotePackage) {
+    if (pkg.metadata.description) {
+      return pkg.metadata.description;
+    }
+  }
+
+  if (error) return <h1>{error.message}</h1>;
+  if (configLoading) return <LoadingIndicator />;
+
+  const loading = !!job;
+
+  return (
+    <SettingSection headingID="config.plugins.available_plugins">
+      <div className="package-manager">
+        <AvailablePackages
+          loading={loading}
+          onInstallPackages={onInstallPackages}
+          renderDescription={renderDescription}
+          loadSource={(source) => loadSource(source)}
+          sources={sources ?? []}
+          addSource={addSource}
+          editSource={editSource}
+          deleteSource={deleteSource}
+        />
+      </div>
+    </SettingSection>
+  );
+};

ui/v2.5/src/components/Settings/ScraperPackageManager.tsx

@@ -0,0 +1,221 @@
+import React, { useEffect, useState, useMemo } from "react";
+import * as GQL from "src/core/generated-graphql";
+import {
+  evictQueries,
+  getClient,
+  queryAvailableScraperPackages,
+  useInstallScraperPackages,
+  useInstalledScraperPackages,
+  useInstalledScraperPackagesStatus,
+  useUninstallScraperPackages,
+  useUpdateScraperPackages,
+} from "src/core/StashService";
+import { useMonitorJob } from "src/utils/job";
+import {
+  AvailablePackages,
+  InstalledPackages,
+  RemotePackage,
+} from "../Shared/PackageManager/PackageManager";
+import { useSettings } from "./context";
+import { LoadingIndicator } from "../Shared/LoadingIndicator";
+import { SettingSection } from "./SettingSection";
+
+const impactedPackageChangeQueries = [
+  GQL.ListPerformerScrapersDocument,
+  GQL.ListSceneScrapersDocument,
+  GQL.ListMovieScrapersDocument,
+  GQL.InstalledScraperPackagesDocument,
+  GQL.InstalledScraperPackagesStatusDocument,
+];
+
+export const InstalledScraperPackages: React.FC = () => {
+  const [loadUpgrades, setLoadUpgrades] = useState(false);
+  const [jobID, setJobID] = useState<string>();
+  const { job } = useMonitorJob(jobID, () => onPackageChanges());
+
+  const { data: installedScrapers, refetch: refetchPackages1 } =
+    useInstalledScraperPackages({
+      skip: loadUpgrades,
+    });
+
+  const {
+    data: withStatus,
+    refetch: refetchPackages2,
+    loading: statusLoading,
+  } = useInstalledScraperPackagesStatus({
+    skip: !loadUpgrades,
+  });
+
+  const [updatePackages] = useUpdateScraperPackages();
+  const [uninstallPackages] = useUninstallScraperPackages();
+
+  async function onUpdatePackages(packages: GQL.PackageSpecInput[]) {
+    const r = await updatePackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.updatePackages);
+  }
+
+  async function onUninstallPackages(packages: GQL.PackageSpecInput[]) {
+    const r = await uninstallPackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.uninstallPackages);
+  }
+
+  function refetchPackages() {
+    refetchPackages1();
+    refetchPackages2();
+  }
+
+  function onPackageChanges() {
+    // job is complete, refresh all local data
+    const ac = getClient();
+    evictQueries(ac.cache, impactedPackageChangeQueries);
+  }
+
+  function onCheckForUpdates() {
+    if (!loadUpgrades) {
+      setLoadUpgrades(true);
+    } else {
+      refetchPackages();
+    }
+  }
+
+  const installedPackages = useMemo(() => {
+    if (withStatus?.installedPackages) {
+      return withStatus.installedPackages;
+    }
+
+    return installedScrapers?.installedPackages ?? [];
+  }, [installedScrapers, withStatus]);
+
+  const loading = !!job || statusLoading;
+
+  return (
+    <SettingSection headingID="config.scraping.installed_scrapers">
+      <div className="package-manager">
+        <InstalledPackages
+          loading={loading}
+          packages={installedPackages}
+          onCheckForUpdates={onCheckForUpdates}
+          onUpdatePackages={(packages) =>
+            onUpdatePackages(
+              packages.map((p) => ({
+                id: p.package_id,
+                sourceURL: p.upgrade!.sourceURL,
+              }))
+            )
+          }
+          onUninstallPackages={(packages) =>
+            onUninstallPackages(
+              packages.map((p) => ({
+                id: p.package_id,
+                sourceURL: p.sourceURL,
+              }))
+            )
+          }
+          updatesLoaded={loadUpgrades}
+        />
+      </div>
+    </SettingSection>
+  );
+};
+
+export const AvailableScraperPackages: React.FC = () => {
+  const { general, loading: configLoading, error, saveGeneral } = useSettings();
+
+  const [sources, setSources] = useState<GQL.PackageSource[]>();
+  const [jobID, setJobID] = useState<string>();
+  const { job } = useMonitorJob(jobID, () => onPackageChanges());
+
+  const [installPackages] = useInstallScraperPackages();
+
+  async function onInstallPackages(packages: GQL.PackageSpecInput[]) {
+    const r = await installPackages({
+      variables: {
+        packages,
+      },
+    });
+
+    setJobID(r.data?.installPackages);
+  }
+
+  function onPackageChanges() {
+    // job is complete, refresh all local data
+    const ac = getClient();
+    evictQueries(ac.cache, impactedPackageChangeQueries);
+  }
+
+  useEffect(() => {
+    if (!sources && !configLoading && general.scraperPackageSources) {
+      setSources(general.scraperPackageSources);
+    }
+  }, [sources, configLoading, general.scraperPackageSources]);
+
+  async function loadSource(source: string): Promise<RemotePackage[]> {
+    const { data } = await queryAvailableScraperPackages(source);
+    return data.availablePackages;
+  }
+
+  function addSource(source: GQL.PackageSource) {
+    saveGeneral({
+      scraperPackageSources: [...(general.scraperPackageSources ?? []), source],
+    });
+
+    setSources((prev) => {
+      return [...(prev ?? []), source];
+    });
+  }
+
+  function editSource(existing: GQL.PackageSource, changed: GQL.PackageSource) {
+    saveGeneral({
+      scraperPackageSources: general.scraperPackageSources?.map((s) =>
+        s.url === existing.url ? changed : s
+      ),
+    });
+
+    setSources((prev) => {
+      return prev?.map((s) => (s.url === existing.url ? changed : s));
+    });
+  }
+
+  function deleteSource(source: GQL.PackageSource) {
+    saveGeneral({
+      scraperPackageSources: general.scraperPackageSources?.filter(
+        (s) => s.url !== source.url
+      ),
+    });
+
+    setSources((prev) => {
+      return prev?.filter((s) => s.url !== source.url);
+    });
+  }
+
+  if (error) return <h1>{error.message}</h1>;
+  if (configLoading) return <LoadingIndicator />;
+
+  const loading = !!job;
+
+  return (
+    <SettingSection headingID="config.scraping.available_scrapers">
+      <div className="package-manager">
+        <AvailablePackages
+          loading={loading}
+          onInstallPackages={onInstallPackages}
+          loadSource={(source) => loadSource(source)}
+          sources={sources ?? []}
+          addSource={addSource}
+          editSource={editSource}
+          deleteSource={deleteSource}
+        />
+      </div>
+    </SettingSection>
+  );
+};

ui/v2.5/src/components/Settings/SettingsPluginsPanel.tsx

@@ -22,6 +22,10 @@ import {
 } from "./Inputs";
 import { faLink, faSyncAlt } from "@fortawesome/free-solid-svg-icons";
 import { useSettings } from "./context";
+import {
+  AvailablePluginPackages,
+  InstalledPluginPackages,
+} from "./PluginPackageManager";
 
 interface IPluginSettingProps {
   pluginID: string;
@@ -242,6 +246,9 @@ export const SettingsPluginsPanel: React.FC = () => {
 
   return (
     <>
+      <InstalledPluginPackages />
+      <AvailablePluginPackages />
+
       <SettingSection headingID="config.categories.plugins">
         <Setting headingID="actions.reload_plugins">
           <Button onClick={() => onReloadPlugins()}>

ui/v2.5/src/components/Settings/SettingsScrapingPanel.tsx

@@ -19,6 +19,10 @@ import { BooleanSetting, StringListSetting, StringSetting } from "./Inputs";
 import { useSettings } from "./context";
 import { StashBoxSetting } from "./StashBoxConfiguration";
 import { faSyncAlt } from "@fortawesome/free-solid-svg-icons";
+import {
+  AvailableScraperPackages,
+  InstalledScraperPackages,
+} from "./ScraperPackageManager";
 
 interface IURLList {
   urls: string[];
@@ -346,6 +350,9 @@ export const SettingsScrapingPanel: React.FC = () => {
         />
       </SettingSection>
 
+      <InstalledScraperPackages />
+      <AvailableScraperPackages />
+
       <SettingSection headingID="config.scraping.scrapers">
         <div className="content">
           <Button onClick={() => onReloadScrapers()}>

ui/v2.5/src/components/Shared/Alert.tsx

@@ -0,0 +1,33 @@
+import React from "react";
+import { Button, Modal } from "react-bootstrap";
+import { FormattedMessage } from "react-intl";
+
+export interface IAlertModalProps {
+  text: JSX.Element | string;
+  show?: boolean;
+  confirmButtonText?: string;
+  onConfirm: () => void;
+  onCancel: () => void;
+}
+
+export const AlertModal: React.FC<IAlertModalProps> = ({
+  text,
+  show,
+  confirmButtonText,
+  onConfirm,
+  onCancel,
+}) => {
+  return (
+    <Modal show={show}>
+      <Modal.Body>{text}</Modal.Body>
+      <Modal.Footer>
+        <Button variant="danger" onClick={() => onConfirm()}>
+          {confirmButtonText ?? <FormattedMessage id="actions.confirm" />}
+        </Button>
+        <Button variant="secondary" onClick={() => onCancel()}>
+          <FormattedMessage id="actions.cancel" />
+        </Button>
+      </Modal.Footer>
+    </Modal>
+  );
+};

ui/v2.5/src/components/Shared/ClearableInput.tsx

@@ -8,17 +8,23 @@ import useFocus from "src/utils/focus";
 interface IClearableInput {
   value: string;
   setValue: (value: string) => void;
-  focus: ReturnType<typeof useFocus>;
+  focus?: ReturnType<typeof useFocus>;
+  placeholder?: string;
 }
 
 export const ClearableInput: React.FC<IClearableInput> = ({
   value,
   setValue,
   focus,
+  placeholder,
 }) => {
   const intl = useIntl();
 
-  const [queryRef, setQueryFocus] = focus;
+  const [defaultQueryRef, setQueryFocusDefault] = useFocus();
+  const [queryRef, setQueryFocus] = focus || [
+    defaultQueryRef,
+    setQueryFocusDefault,
+  ];
   const queryClearShowing = !!value;
 
   function onChangeQuery(event: React.FormEvent<HTMLInputElement>) {
@@ -34,7 +40,7 @@ export const ClearableInput: React.FC<IClearableInput> = ({
     <div className="clearable-input-group">
       <FormControl
         ref={queryRef}
-        placeholder={`${intl.formatMessage({ id: "actions.search" })}…`}
+        placeholder={placeholder}
         value={value}
         onInput={onChangeQuery}
         className="clearable-text-field"

ui/v2.5/src/components/Shared/Modal.tsx

@@ -13,7 +13,7 @@ interface IButton {
 interface IModal {
   show: boolean;
   onHide?: () => void;
-  header?: string;
+  header?: JSX.Element | string;
   icon?: IconDefinition;
   cancel?: IButton;
   accept?: IButton;
@@ -59,24 +59,6 @@ export const ModalComponent: React.FC<IModal> = ({
       <div>{leftFooterButtons}</div>
       <div>
         {footerButtons}
-        {cancel ? (
-          <Button
-            disabled={isRunning}
-            variant={cancel.variant ?? "primary"}
-            onClick={cancel.onClick}
-            className="ml-2"
-          >
-            {cancel.text ?? (
-              <FormattedMessage
-                id="actions.cancel"
-                defaultMessage="Cancel"
-                description="Cancels the current action and dismisses the modal."
-              />
-            )}
-          </Button>
-        ) : (
-          ""
-        )}
         <Button
           disabled={isRunning || disabled}
           variant={accept?.variant ?? "primary"}
@@ -95,6 +77,24 @@ export const ModalComponent: React.FC<IModal> = ({
             )
           )}
         </Button>
+        {cancel ? (
+          <Button
+            disabled={isRunning}
+            variant={cancel.variant ?? "primary"}
+            onClick={cancel.onClick}
+            className="ml-2"
+          >
+            {cancel.text ?? (
+              <FormattedMessage
+                id="actions.cancel"
+                defaultMessage="Cancel"
+                description="Cancels the current action and dismisses the modal."
+              />
+            )}
+          </Button>
+        ) : (
+          ""
+        )}
       </div>
     </Modal.Footer>
   </Modal>

ui/v2.5/src/components/Shared/PackageManager/styles.scss

@@ -0,0 +1,100 @@
+.package-manager {
+  padding: 1em;
+
+  .package-source {
+    font-weight: bold;
+
+    .source-controls {
+      align-items: center;
+      display: flex;
+      justify-content: end;
+    }
+  }
+
+  .package-cell,
+  .package-source {
+    cursor: pointer;
+  }
+
+  .package-collapse-button {
+    color: $text-color;
+  }
+
+  .package-manager-table-container {
+    max-height: 300px;
+    overflow-y: auto;
+  }
+
+  table thead {
+    background-color: $card-bg;
+    position: sticky;
+    top: 0;
+    z-index: 1;
+
+    .button-cell {
+      width: 40px;
+    }
+  }
+
+  table td {
+    vertical-align: middle;
+  }
+
+  .package-version,
+  .package-date,
+  .package-name,
+  .package-id {
+    display: block;
+  }
+
+  .package-date,
+  .package-id {
+    color: $muted-gray;
+    font-size: 0.8rem;
+  }
+
+  .package-manager-toolbar {
+    display: flex;
+    justify-content: space-between;
+
+    div {
+      display: flex;
+    }
+
+    .btn {
+      margin-left: 0.5em;
+    }
+  }
+
+  .package-required-by {
+    color: $warning;
+    font-size: 0.8rem;
+  }
+
+  .LoadingIndicator-message {
+    display: inline-block;
+    font-size: 1rem;
+    margin-left: 0.5em;
+    margin-top: 0;
+  }
+
+  .source-error {
+    & > .fa-icon {
+      color: $warning;
+    }
+
+    .btn {
+      margin-left: 0.5em;
+    }
+  }
+}
+
+.package-manager-no-results {
+  color: $text-muted;
+  padding: 1em;
+  text-align: center;
+
+  .btn {
+    margin-top: 0.5em;
+  }
+}

ui/v2.5/src/components/Shared/styles.scss

@@ -457,21 +457,31 @@ div.react-datepicker {
 .clearable-text-field,
 .clearable-text-field:active,
 .clearable-text-field:focus {
-  background-color: #394b59;
+  background-color: $secondary;
   border: 0;
-  border-color: #394b59;
+  border-color: $secondary;
   color: #fff;
 }
 
 .clearable-text-field-clear {
-  background-color: #394b59;
-  color: #bfccd6;
+  background-color: $secondary;
+  color: $muted-gray;
   font-size: 0.875rem;
   margin: 0.375rem 0.75rem;
   padding: 0;
   position: absolute;
   right: 0;
   z-index: 4;
+
+  &:hover,
+  &:focus,
+  &:active,
+  &:not(:disabled):not(.disabled):active,
+  &:not(:disabled):not(.disabled):active:focus {
+    background-color: $secondary;
+    border-color: transparent;
+    box-shadow: none;
+  }
 }
 
 .string-list-row .input-group {