import React, { useMemo, useState } from "react"; import { Button, ButtonGroup, Card, Col, Dropdown, Form, OverlayTrigger, Row, Table, Tooltip, } from "react-bootstrap"; import { Link, useHistory } from "react-router-dom"; import { FormattedMessage, FormattedNumber, useIntl } from "react-intl"; import * as GQL from "src/core/generated-graphql"; import { LoadingIndicator } from "../Shared/LoadingIndicator"; import { ErrorMessage } from "../Shared/ErrorMessage"; import { HoverPopover } from "../Shared/HoverPopover"; import { Icon } from "../Shared/Icon"; import { GalleryLink, MovieLink, SceneMarkerLink, TagLink, } from "../Shared/TagLink"; import { SweatDrops } from "../Shared/SweatDrops"; import { Pagination } from "src/components/List/Pagination"; import TextUtils from "src/utils/text"; import { DeleteScenesDialog } from "src/components/Scenes/DeleteScenesDialog"; import { EditScenesDialog } from "../Scenes/EditScenesDialog"; import { PerformerPopoverButton } from "../Shared/PerformerPopoverButton"; import { faBox, faExclamationTriangle, faFileAlt, faFilm, faImages, faMapMarkerAlt, faPencilAlt, faTag, faTrash, } from "@fortawesome/free-solid-svg-icons"; import { SceneMergeModal } from "../Scenes/SceneMergeDialog"; import { objectTitle } from "src/core/files"; const CLASSNAME = "duplicate-checker"; const defaultDurationDiff = "1"; export const SceneDuplicateChecker: React.FC = () => { const intl = useIntl(); const history = useHistory(); const query = new URLSearchParams(history.location.search); const currentPage = Number.parseInt(query.get("page") ?? "1", 10); const pageSize = Number.parseInt(query.get("size") ?? "20", 10); const hashDistance = Number.parseInt(query.get("distance") ?? "0", 10); const durationDiff = Number.parseFloat( query.get("durationDiff") ?? defaultDurationDiff ); const [currentPageSize, setCurrentPageSize] = useState(pageSize); const [isMultiDelete, setIsMultiDelete] = useState(false); const [deletingScenes, setDeletingScenes] = useState(false); const [editingScenes, setEditingScenes] = useState(false); const [chkSafeSelect, setChkSafeSelect] = useState(true); const [checkedScenes, setCheckedScenes] = useState>( {} ); const { data, loading, refetch } = GQL.useFindDuplicateScenesQuery({ fetchPolicy: "no-cache", variables: { distance: hashDistance, duration_diff: durationDiff, }, }); const scenes = data?.findDuplicateScenes ?? []; const { data: missingPhash } = GQL.useFindScenesQuery({ variables: { filter: { per_page: 0, }, scene_filter: { is_missing: "phash", file_count: { modifier: GQL.CriterionModifier.GreaterThan, value: 0, }, }, }, }); const [selectedScenes, setSelectedScenes] = useState< GQL.SlimSceneDataFragment[] | null >(null); const [mergeScenes, setMergeScenes] = useState<{ id: string; title: string }[]>(); const pageOptions = useMemo(() => { const pageSizes = [ 10, 20, 30, 40, 50, 100, 150, 200, 250, 500, 750, 1000, 1250, 1500, ]; const filteredSizes = pageSizes.filter((s, i) => { return scenes.length > s || i == 0 || scenes.length > pageSizes[i - 1]; }); return filteredSizes.map((size) => { return ( ); }); }, [scenes.length]); if (loading) return ; if (!data) return ; const filteredScenes = scenes.slice( (currentPage - 1) * pageSize, currentPage * pageSize ); const checkCount = Object.keys(checkedScenes).filter( (id) => checkedScenes[id] ).length; const setQuery = (q: Record) => { const newQuery = new URLSearchParams(query); for (const key of Object.keys(q)) { const value = q[key]; if (value !== undefined) { newQuery.set(key, String(value)); } else { newQuery.delete(key); } } history.push({ search: newQuery.toString() }); }; const resetCheckboxSelection = () => { const updatedScenes: Record = {}; Object.keys(checkedScenes).forEach((sceneKey) => { updatedScenes[sceneKey] = false; }); setCheckedScenes(updatedScenes); }; function onDeleteDialogClosed(deleted: boolean) { setDeletingScenes(false); if (deleted) { setSelectedScenes(null); refetch(); if (isMultiDelete) setCheckedScenes({}); } resetCheckboxSelection(); } const findLargestScene = (group: GQL.SlimSceneDataFragment[]) => { // Get total size of a scene const totalSize = (scene: GQL.SlimSceneDataFragment) => { return scene.files.reduce((sum: number, f) => sum + (f.size || 0), 0); }; // Find scene object with maximum total size return group.reduce((largest, scene) => { const largestSize = totalSize(largest); const currentSize = totalSize(scene); return currentSize > largestSize ? scene : largest; }); }; // Helper to get file date const findFirstFileByAge = ( oldest: boolean, compareScenes: GQL.SlimSceneDataFragment[] ) => { let selectedFile: GQL.VideoFileDataFragment; let oldestTimestamp: Date | undefined = undefined; // Loop through all files for (const file of compareScenes.flatMap((s) => s.files)) { // Get timestamp const timestamp: Date = new Date(file.mod_time); // Check if current file is oldest if (oldest) { if (oldestTimestamp === undefined || timestamp < oldestTimestamp) { oldestTimestamp = timestamp; selectedFile = file; } } else { if (oldestTimestamp === undefined || timestamp > oldestTimestamp) { oldestTimestamp = timestamp; selectedFile = file; } } } // Find scene with oldest file return compareScenes.find((s) => s.files.some((f) => f.id === selectedFile.id) ); }; function checkSameCodec(codecGroup: GQL.SlimSceneDataFragment[]) { const codecs = codecGroup.map((s) => s.files[0]?.video_codec); return new Set(codecs).size === 1; } const onSelectLargestClick = () => { setSelectedScenes([]); const checkedArray: Record = {}; filteredScenes.forEach((group) => { if (chkSafeSelect && !checkSameCodec(group)) { return; } // Find largest scene in group a const largest = findLargestScene(group); group.forEach((scene) => { if (scene !== largest) { checkedArray[scene.id] = true; } }); }); setCheckedScenes(checkedArray); }; const onSelectByAge = (oldest: boolean) => { setSelectedScenes([]); const checkedArray: Record = {}; filteredScenes.forEach((group) => { if (chkSafeSelect && !checkSameCodec(group)) { return; } const oldestScene = findFirstFileByAge(oldest, group); group.forEach((scene) => { if (scene !== oldestScene) { checkedArray[scene.id] = true; } }); }); setCheckedScenes(checkedArray); }; const handleCheck = (checked: boolean, sceneID: string) => { setCheckedScenes({ ...checkedScenes, [sceneID]: checked }); }; const handleDeleteChecked = () => { setSelectedScenes(scenes.flat().filter((s) => checkedScenes[s.id])); setDeletingScenes(true); setIsMultiDelete(true); }; const handleDeleteScene = (scene: GQL.SlimSceneDataFragment) => { setSelectedScenes([scene]); setDeletingScenes(true); setIsMultiDelete(false); }; function onEdit() { setSelectedScenes(scenes.flat().filter((s) => checkedScenes[s.id])); setEditingScenes(true); resetCheckboxSelection(); } const renderFilesize = (filesize: number | null | undefined) => { const { size: parsedSize, unit } = TextUtils.fileSize(filesize ?? 0); return ( ); }; function maybeRenderMissingPhashWarning() { const missingPhashes = missingPhash?.findScenes.count ?? 0; if (missingPhashes > 0) { return (

Missing phashes for {missingPhashes} scenes. Please run the phash generation task.

); } } function maybeRenderEdit() { if (editingScenes && selectedScenes) { return ( setEditingScenes(false)} /> ); } } function maybeRenderTagPopoverButton(scene: GQL.SlimSceneDataFragment) { if (scene.tags.length <= 0) return; const popoverContent = scene.tags.map((tag) => ( )); return ( ); } function maybeRenderPerformerPopoverButton(scene: GQL.SlimSceneDataFragment) { if (scene.performers.length <= 0) return; return ; } function maybeRenderMoviePopoverButton(scene: GQL.SlimSceneDataFragment) { if (scene.movies.length <= 0) return; const popoverContent = scene.movies.map((sceneMovie) => (
{sceneMovie.movie.name
)); return ( ); } function maybeRenderSceneMarkerPopoverButton( scene: GQL.SlimSceneDataFragment ) { if (scene.scene_markers.length <= 0) return; const popoverContent = scene.scene_markers.map((marker) => { const markerWithScene = { ...marker, scene: { id: scene.id } }; return ; }); return ( ); } function maybeRenderOCounter(scene: GQL.SlimSceneDataFragment) { if (scene.o_counter) { return (
); } } function maybeRenderGallery(scene: GQL.SlimSceneDataFragment) { if (scene.galleries.length <= 0) return; const popoverContent = scene.galleries.map((gallery) => ( )); return ( ); } function maybeRenderFileCount(scene: GQL.SlimSceneDataFragment) { if (scene.files.length <= 1) return; const popoverContent = ( ); return ( ); } function maybeRenderOrganized(scene: GQL.SlimSceneDataFragment) { if (scene.organized) { return (
); } } function maybeRenderPopoverButtonGroup(scene: GQL.SlimSceneDataFragment) { if ( scene.tags.length > 0 || scene.performers.length > 0 || scene.movies.length > 0 || scene.scene_markers.length > 0 || scene?.o_counter || scene.galleries.length > 0 || scene.files.length > 1 || scene.organized ) { return ( <> {maybeRenderTagPopoverButton(scene)} {maybeRenderPerformerPopoverButton(scene)} {maybeRenderMoviePopoverButton(scene)} {maybeRenderSceneMarkerPopoverButton(scene)} {maybeRenderOCounter(scene)} {maybeRenderGallery(scene)} {maybeRenderFileCount(scene)} {maybeRenderOrganized(scene)} ); } } function renderPagination() { return (
{checkCount > 0 && ( {intl.formatMessage({ id: "actions.edit" })} } > {intl.formatMessage({ id: "actions.delete" })} } > )} { setQuery({ page: newPage === 1 ? undefined : newPage }); resetCheckboxSelection(); }} /> { setCurrentPageSize(parseInt(e.currentTarget.value, 10)); setQuery({ size: e.currentTarget.value === "20" ? undefined : e.currentTarget.value, }); resetCheckboxSelection(); }} > {pageOptions}
); } function renderMergeDialog() { if (mergeScenes) { return ( { setMergeScenes(undefined); if (mergedID) { // refresh refetch(); } }} show /> ); } } function onMergeClicked( sceneGroup: GQL.SlimSceneDataFragment[], scene: GQL.SlimSceneDataFragment ) { const selected = scenes.flat().filter((s) => checkedScenes[s.id]); // if scenes in this group other than this scene are selected, then only // the selected scenes will be selected as source. Otherwise all other // scenes will be source let srcScenes = selected.filter((s) => { if (s === scene) return false; return sceneGroup.includes(s); }) ?? []; if (!srcScenes.length) { srcScenes = sceneGroup.filter((s) => s !== scene); } // insert subject scene to the front so that it is considered the destination srcScenes.unshift(scene); setMergeScenes( srcScenes.map((s) => { return { id: s.id, title: objectTitle(s), }; }) ); } return (
{deletingScenes && selectedScenes && ( )} {renderMergeDialog()} {maybeRenderEdit()}

setQuery({ distance: e.currentTarget.value === "0" ? undefined : e.currentTarget.value, page: undefined, }) } defaultValue={hashDistance} className="input-control ml-4" > setQuery({ durationDiff: e.currentTarget.value === defaultDurationDiff ? undefined : e.currentTarget.value, page: undefined, }) } defaultValue={durationDiff} className="input-control ml-4" > resetCheckboxSelection()}> {intl.formatMessage({ id: "dupe_check.select_none" })} onSelectLargestClick()}> {intl.formatMessage({ id: "dupe_check.select_all_but_largest_file", })} onSelectByAge(true)}> {intl.formatMessage({ id: "dupe_check.select_oldest", })} onSelectByAge(false)}> {intl.formatMessage({ id: "dupe_check.select_youngest", })} { setChkSafeSelect(e.target.checked); resetCheckboxSelection(); }} />
{maybeRenderMissingPhashWarning()} {renderPagination()} {filteredScenes.map((group, groupIndex) => group.map((scene, i) => { const file = scene.files.length > 0 ? scene.files[0] : undefined; return ( <> {i === 0 && groupIndex !== 0 ? ( ) : undefined} ); }) )}
{intl.formatMessage({ id: "details" })} {intl.formatMessage({ id: "duration" })} {intl.formatMessage({ id: "filesize" })} {intl.formatMessage({ id: "resolution" })} {intl.formatMessage({ id: "bitrate" })} {intl.formatMessage({ id: "media_info.video_codec" })} {intl.formatMessage({ id: "actions.delete" })}
handleCheck(e.currentTarget.checked, scene.id) } /> } placement="right" >

{" "} {scene.title ? scene.title : TextUtils.fileNameFromPath( file?.path ?? "" )}{" "}

{file?.path ?? ""}

{maybeRenderPopoverButtonGroup(scene)} {file?.duration && TextUtils.secondsToTimestamp(file.duration)} {renderFilesize(file?.size ?? 0)} {`${file?.width ?? 0}x${file?.height ?? 0}`}  mbps {file?.video_codec ?? ""}
{scenes.length === 0 && (

No duplicates found.

)} {renderPagination()}
); }; export default SceneDuplicateChecker;