Selective clean (#2125)

* Add backend support for selective clean
* Add selective clean button and dialog

graphql/schema/types/metadata.graphql

@@ -106,6 +106,8 @@ type ScanMetadataOptions {
 }
 
 input CleanMetadataInput {
+  paths: [String!]
+  
   """Do a dry run. Don't delete any files"""
   dryRun: Boolean!
 }

pkg/gallery/filter.go

@@ -0,0 +1,40 @@
+package gallery
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func PathsFilter(paths []string) *models.GalleryFilterType {
+	if paths == nil {
+		return nil
+	}
+
+	sep := string(filepath.Separator)
+
+	var ret *models.GalleryFilterType
+	var or *models.GalleryFilterType
+	for _, p := range paths {
+		newOr := &models.GalleryFilterType{}
+		if or != nil {
+			or.Or = newOr
+		} else {
+			ret = newOr
+		}
+
+		or = newOr
+
+		if !strings.HasSuffix(p, sep) {
+			p += sep
+		}
+
+		or.Path = &models.StringCriterionInput{
+			Modifier: models.CriterionModifierEquals,
+			Value:    p + "%",
+		}
+	}
+
+	return ret
+}

pkg/image/filter.go

@@ -0,0 +1,40 @@
+package image
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func PathsFilter(paths []string) *models.ImageFilterType {
+	if paths == nil {
+		return nil
+	}
+
+	sep := string(filepath.Separator)
+
+	var ret *models.ImageFilterType
+	var or *models.ImageFilterType
+	for _, p := range paths {
+		newOr := &models.ImageFilterType{}
+		if or != nil {
+			or.Or = newOr
+		} else {
+			ret = newOr
+		}
+
+		or = newOr
+
+		if !strings.HasSuffix(p, sep) {
+			p += sep
+		}
+
+		or.Path = &models.StringCriterionInput{
+			Modifier: models.CriterionModifierEquals,
+			Value:    p + "%",
+		}
+	}
+
+	return ret
+}

pkg/manager/task_clean.go

@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/stashapp/stash/pkg/file"
+	"github.com/stashapp/stash/pkg/gallery"
 	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/job"
 	"github.com/stashapp/stash/pkg/logger"
@@ -66,28 +67,35 @@ func (j *cleanJob) Execute(ctx context.Context, progress *job.Progress) {
 }
 
 func (j *cleanJob) getCount(r models.ReaderRepository) (int, error) {
-	sceneCount, err := r.Scene().Count()
+	sceneFilter := scene.PathsFilter(j.input.Paths)
+	sceneResult, err := r.Scene().Query(models.SceneQueryOptions{
+		QueryOptions: models.QueryOptions{
+			Count: true,
+		},
+		SceneFilter: sceneFilter,
+	})
 	if err != nil {
 		return 0, err
 	}
 
-	imageCount, err := r.Image().Count()
+	imageCount, err := r.Image().QueryCount(image.PathsFilter(j.input.Paths), nil)
 	if err != nil {
 		return 0, err
 	}
 
-	galleryCount, err := r.Gallery().Count()
+	galleryCount, err := r.Gallery().QueryCount(gallery.PathsFilter(j.input.Paths), nil)
 	if err != nil {
 		return 0, err
 	}
 
-	return sceneCount + imageCount + galleryCount, nil
+	return sceneResult.Count + imageCount + galleryCount, nil
 }
 
 func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb models.SceneReader) error {
 	batchSize := 1000
 
 	findFilter := models.BatchFindFilter(batchSize)
+	sceneFilter := scene.PathsFilter(j.input.Paths)
 	sort := "path"
 	findFilter.Sort = &sort
 
@@ -99,7 +107,7 @@ func (j *cleanJob) processScenes(ctx context.Context, progress *job.Progress, qb
 			return nil
 		}
 
-		scenes, err := scene.Query(qb, nil, findFilter)
+		scenes, err := scene.Query(qb, sceneFilter, findFilter)
 		if err != nil {
 			return fmt.Errorf("error querying for scenes: %w", err)
 		}
@@ -150,6 +158,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress,
 	batchSize := 1000
 
 	findFilter := models.BatchFindFilter(batchSize)
+	galleryFilter := gallery.PathsFilter(j.input.Paths)
 	sort := "path"
 	findFilter.Sort = &sort
 
@@ -161,7 +170,7 @@ func (j *cleanJob) processGalleries(ctx context.Context, progress *job.Progress,
 			return nil
 		}
 
-		galleries, _, err := qb.Query(nil, findFilter)
+		galleries, _, err := qb.Query(galleryFilter, findFilter)
 		if err != nil {
 			return fmt.Errorf("error querying for galleries: %w", err)
 		}
@@ -210,6 +219,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb
 	batchSize := 1000
 
 	findFilter := models.BatchFindFilter(batchSize)
+	imageFilter := image.PathsFilter(j.input.Paths)
 
 	// performance consideration: order by path since default ordering by
 	// title is slow
@@ -224,7 +234,7 @@ func (j *cleanJob) processImages(ctx context.Context, progress *job.Progress, qb
 			return nil
 		}
 
-		images, err := image.Query(qb, nil, findFilter)
+		images, err := image.Query(qb, imageFilter, findFilter)
 		if err != nil {
 			return fmt.Errorf("error querying for images: %w", err)
 		}

pkg/match/path.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strings"
 
+	"github.com/stashapp/stash/pkg/gallery"
 	"github.com/stashapp/stash/pkg/image"
 	"github.com/stashapp/stash/pkg/models"
 	"github.com/stashapp/stash/pkg/scene"
@@ -175,38 +176,6 @@ func PathToTags(path string, tagReader models.TagReader) ([]*models.Tag, error)
 	return ret, nil
 }
 
-func scenePathsFilter(paths []string) *models.SceneFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.SceneFilterType
-	var or *models.SceneFilterType
-	for _, p := range paths {
-		newOr := &models.SceneFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p += sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
 func PathToScenes(name string, paths []string, sceneReader models.SceneReader) ([]*models.Scene, error) {
 	regex := getPathQueryRegex(name)
 	organized := false
@@ -218,7 +187,7 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) (
 		Organized: &organized,
 	}
 
-	filter.And = scenePathsFilter(paths)
+	filter.And = scene.PathsFilter(paths)
 
 	pp := models.PerPageAll
 	scenes, err := scene.Query(sceneReader, &filter, &models.FindFilterType{
@@ -239,38 +208,6 @@ func PathToScenes(name string, paths []string, sceneReader models.SceneReader) (
 	return ret, nil
 }
 
-func imagePathsFilter(paths []string) *models.ImageFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.ImageFilterType
-	var or *models.ImageFilterType
-	for _, p := range paths {
-		newOr := &models.ImageFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p += sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
 func PathToImages(name string, paths []string, imageReader models.ImageReader) ([]*models.Image, error) {
 	regex := getPathQueryRegex(name)
 	organized := false
@@ -282,7 +219,7 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) (
 		Organized: &organized,
 	}
 
-	filter.And = imagePathsFilter(paths)
+	filter.And = image.PathsFilter(paths)
 
 	pp := models.PerPageAll
 	images, err := image.Query(imageReader, &filter, &models.FindFilterType{
@@ -303,38 +240,6 @@ func PathToImages(name string, paths []string, imageReader models.ImageReader) (
 	return ret, nil
 }
 
-func galleryPathsFilter(paths []string) *models.GalleryFilterType {
-	if paths == nil {
-		return nil
-	}
-
-	sep := string(filepath.Separator)
-
-	var ret *models.GalleryFilterType
-	var or *models.GalleryFilterType
-	for _, p := range paths {
-		newOr := &models.GalleryFilterType{}
-		if or != nil {
-			or.Or = newOr
-		} else {
-			ret = newOr
-		}
-
-		or = newOr
-
-		if !strings.HasSuffix(p, sep) {
-			p += sep
-		}
-
-		or.Path = &models.StringCriterionInput{
-			Modifier: models.CriterionModifierEquals,
-			Value:    p + "%",
-		}
-	}
-
-	return ret
-}
-
 func PathToGalleries(name string, paths []string, galleryReader models.GalleryReader) ([]*models.Gallery, error) {
 	regex := getPathQueryRegex(name)
 	organized := false
@@ -346,7 +251,7 @@ func PathToGalleries(name string, paths []string, galleryReader models.GalleryRe
 		Organized: &organized,
 	}
 
-	filter.And = galleryPathsFilter(paths)
+	filter.And = gallery.PathsFilter(paths)
 
 	pp := models.PerPageAll
 	gallerys, _, err := galleryReader.Query(&filter, &models.FindFilterType{

pkg/scene/filter.go

@@ -0,0 +1,40 @@
+package scene
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/stashapp/stash/pkg/models"
+)
+
+func PathsFilter(paths []string) *models.SceneFilterType {
+	if paths == nil {
+		return nil
+	}
+
+	sep := string(filepath.Separator)
+
+	var ret *models.SceneFilterType
+	var or *models.SceneFilterType
+	for _, p := range paths {
+		newOr := &models.SceneFilterType{}
+		if or != nil {
+			or.Or = newOr
+		} else {
+			ret = newOr
+		}
+
+		or = newOr
+
+		if !strings.HasSuffix(p, sep) {
+			p += sep
+		}
+
+		or.Path = &models.StringCriterionInput{
+			Modifier: models.CriterionModifierEquals,
+			Value:    p + "%",
+		}
+	}
+
+	return ret
+}

ui/v2.5/src/components/Changelog/versions/v0120.md

@@ -1,8 +1,9 @@
 ### ✨ New Features
+* Added selective clean task. ([#2125](https://github.com/stashapp/stash/pull/2125))
 * Show heatmaps and median stroke speed for interactive scenes on the scenes page. ([#2096](https://github.com/stashapp/stash/pull/2096))
 * Save task options when scanning, generating and auto-tagging. ([#1949](https://github.com/stashapp/stash/pull/1949), [#2061](https://github.com/stashapp/stash/pull/2061))
 * Changed query string parsing behaviour to require all words by default, with the option to `or` keywords and exclude keywords. See the `Browsing` section of the manual for details. ([#1982](https://github.com/stashapp/stash/pull/1982))
-* Add forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973))
+* Added forward jump 10 second button to video player. ([#1973](https://github.com/stashapp/stash/pull/1973))
 
 ### 🎨 Improvements
 * Added keyboard shortcuts to hide scene page sidebar and scene scrubber. ([#2099](https://github.com/stashapp/stash/pull/2099))

ui/v2.5/src/components/Settings/Tasks/DataManagementTasks.tsx

@@ -1,6 +1,6 @@
 import React, { useState } from "react";
 import { FormattedMessage, useIntl } from "react-intl";
-import { Button, Form } from "react-bootstrap";
+import { Button, Col, Form, Row } from "react-bootstrap";
 import {
   mutateMigrateHashNaming,
   mutateMetadataExport,
@@ -17,6 +17,104 @@ import { SettingSection } from "../SettingSection";
 import { BooleanSetting, Setting } from "../Inputs";
 import { ManualLink } from "src/components/Help/Manual";
 import { Icon } from "src/components/Shared";
+import { ConfigurationContext } from "src/hooks/Config";
+import { FolderSelect } from "src/components/Shared/FolderSelect/FolderSelect";
+
+interface ICleanDialog {
+  pathSelection?: boolean;
+  dryRun: boolean;
+  onClose: (paths?: string[]) => void;
+}
+
+const CleanDialog: React.FC<ICleanDialog> = ({
+  pathSelection = false,
+  dryRun,
+  onClose,
+}) => {
+  const intl = useIntl();
+  const { configuration } = React.useContext(ConfigurationContext);
+
+  const libraryPaths = configuration?.general.stashes.map((s) => s.path);
+
+  const [paths, setPaths] = useState<string[]>([]);
+  const [currentDirectory, setCurrentDirectory] = useState<string>("");
+
+  function removePath(p: string) {
+    setPaths(paths.filter((path) => path !== p));
+  }
+
+  function addPath(p: string) {
+    if (p && !paths.includes(p)) {
+      setPaths(paths.concat(p));
+    }
+  }
+
+  let msg;
+  if (dryRun) {
+    msg = (
+      <p>{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}</p>
+    );
+  } else {
+    msg = (
+      <p>{intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}</p>
+    );
+  }
+
+  return (
+    <Modal
+      show
+      icon="trash-alt"
+      disabled={pathSelection && paths.length === 0}
+      accept={{
+        text: intl.formatMessage({ id: "actions.clean" }),
+        variant: "danger",
+        onClick: () => onClose(paths),
+      }}
+      cancel={{ onClick: () => onClose() }}
+    >
+      <div className="dialog-container">
+        <div className="mb-3">
+          {paths.map((p) => (
+            <Row className="align-items-center mb-1" key={p}>
+              <Form.Label column xs={10}>
+                {p}
+              </Form.Label>
+              <Col xs={2} className="d-flex justify-content-end">
+                <Button
+                  className="ml-auto"
+                  size="sm"
+                  variant="danger"
+                  title={intl.formatMessage({ id: "actions.delete" })}
+                  onClick={() => removePath(p)}
+                >
+                  <Icon icon="minus" />
+                </Button>
+              </Col>
+            </Row>
+          ))}
+
+          {pathSelection ? (
+            <FolderSelect
+              currentDirectory={currentDirectory}
+              setCurrentDirectory={(v) => setCurrentDirectory(v)}
+              defaultDirectories={libraryPaths}
+              appendButton={
+                <Button
+                  variant="secondary"
+                  onClick={() => addPath(currentDirectory)}
+                >
+                  <Icon icon="plus" />
+                </Button>
+              }
+            />
+          ) : undefined}
+        </div>
+
+        {msg}
+      </div>
+    </Modal>
+  );
+};
 
 interface ICleanOptions {
   options: GQL.CleanMetadataInput;
@@ -56,6 +154,7 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     importAlert: false,
     import: false,
     clean: false,
+    cleanAlert: false,
   });
 
   const [cleanOptions, setCleanOptions] = useState<GQL.CleanMetadataInput>({
@@ -110,39 +209,12 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     return <ImportDialog onClose={() => setDialogOpen({ import: false })} />;
   }
 
-  function renderCleanDialog() {
+  async function onClean(paths?: string[]) {
-    let msg;
-    if (cleanOptions.dryRun) {
-      msg = (
-        <p>{intl.formatMessage({ id: "actions.tasks.dry_mode_selected" })}</p>
-      );
-    } else {
-      msg = (
-        <p>
-          {intl.formatMessage({ id: "actions.tasks.clean_confirm_message" })}
-        </p>
-      );
-    }
-
-    return (
-      <Modal
-        show={dialogOpen.clean}
-        icon="trash-alt"
-        accept={{
-          text: intl.formatMessage({ id: "actions.clean" }),
-          variant: "danger",
-          onClick: onClean,
-        }}
-        cancel={{ onClick: () => setDialogOpen({ clean: false }) }}
-      >
-        {msg}
-      </Modal>
-    );
-  }
-
-  async function onClean() {
     try {
-      await mutateMetadataClean(cleanOptions);
+      await mutateMetadataClean({
+        ...cleanOptions,
+        paths,
+      });
 
       Toast.success({
         content: intl.formatMessage(
@@ -212,7 +284,30 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
     <Form.Group>
       {renderImportAlert()}
       {renderImportDialog()}
-      {renderCleanDialog()}
+      {dialogOpen.cleanAlert || dialogOpen.clean ? (
+        <CleanDialog
+          dryRun={cleanOptions.dryRun}
+          pathSelection={dialogOpen.clean}
+          onClose={(p) => {
+            // undefined means cancelled
+            if (p !== undefined) {
+              if (dialogOpen.cleanAlert) {
+                // don't provide paths
+                onClean();
+              } else {
+                onClean(p);
+              }
+            }
+
+            setDialogOpen({
+              clean: false,
+              cleanAlert: false,
+            });
+          }}
+        />
+      ) : (
+        dialogOpen.clean
+      )}
 
       <SettingSection headingID="config.tasks.maintenance">
         <div className="setting-group">
@@ -230,10 +325,17 @@ export const DataManagementTasks: React.FC<IDataManagementTasks> = ({
             <Button
               variant="danger"
               type="submit"
-              onClick={() => setDialogOpen({ clean: true })}
+              onClick={() => setDialogOpen({ cleanAlert: true })}
             >
               <FormattedMessage id="actions.clean" />…
             </Button>
+            <Button
+              variant="danger"
+              type="submit"
+              onClick={() => setDialogOpen({ clean: true })}
+            >
+              <FormattedMessage id="actions.selective_clean" />…
+            </Button>
           </Setting>
           <CleanOptions
             options={cleanOptions}

ui/v2.5/src/locales/en-GB.json

@@ -79,6 +79,7 @@
     "select_folders": "Select folders",
     "select_none": "Select None",
     "selective_auto_tag": "Selective Auto Tag",
+    "selective_clean": "Selective Clean",
     "selective_scan": "Selective Scan",
     "set_as_default": "Set as default",
     "set_back_image": "Back image…",