Preview scrubber (#4022)

* Add sprite info hook
* Remove axios dependency
* Add preview scrubber
* Add scrubber timestamp
* On click go to timestamp

ui/v2.5/package.json

@@ -32,7 +32,6 @@
     "@silvermine/videojs-airplay": "^1.2.0",
     "@silvermine/videojs-chromecast": "^1.4.1",
     "apollo-upload-client": "^17.0.0",
-    "axios": "^1.3.3",
     "base64-blob": "^1.4.1",
     "bootstrap": "^4.6.2",
     "classnames": "^2.3.2",

ui/v2.5/src/components/ScenePlayer/ScenePlayerScrubber.tsx

@@ -6,15 +6,14 @@ import React, {
   useCallback,
 } from "react";
 import { Button } from "react-bootstrap";
-import axios from "axios";
 import * as GQL from "src/core/generated-graphql";
 import TextUtils from "src/utils/text";
-import { WebVTT } from "videojs-vtt.js";
 import { Icon } from "src/components/Shared/Icon";
 import {
   faChevronRight,
   faChevronLeft,
 } from "@fortawesome/free-solid-svg-icons";
+import { useSpriteInfo } from "src/hooks/sprite";
 
 interface IScenePlayerScrubberProps {
   file: GQL.VideoFileDataFragment;
@@ -29,42 +28,6 @@ interface ISceneSpriteItem {
   time: string;
 }
 
-interface ISceneSpriteInfo {
-  url: string;
-  start: number;
-  end: number;
-  x: number;
-  y: number;
-  w: number;
-  h: number;
-}
-
-async function fetchSpriteInfo(vttPath: string) {
-  const response = await axios.get<string>(vttPath, { responseType: "text" });
-
-  const sprites: ISceneSpriteInfo[] = [];
-
-  const parser = new WebVTT.Parser(window, WebVTT.StringDecoder());
-  parser.oncue = (cue: VTTCue) => {
-    const match = cue.text.match(/^([^#]*)#xywh=(\d+),(\d+),(\d+),(\d+)$/i);
-    if (!match) return;
-
-    sprites.push({
-      url: new URL(match[1], vttPath).href,
-      start: cue.startTime,
-      end: cue.endTime,
-      x: Number(match[2]),
-      y: Number(match[3]),
-      w: Number(match[4]),
-      h: Number(match[5]),
-    });
-  };
-  parser.parse(response.data);
-  parser.flush();
-
-  return sprites;
-}
-
 export const ScenePlayerScrubber: React.FC<IScenePlayerScrubberProps> = ({
   file,
   scene,
@@ -119,34 +82,32 @@ export const ScenePlayerScrubber: React.FC<IScenePlayerScrubberProps> = ({
     [onSeek, file.duration, scrubWidth]
   );
 
+  const spriteInfo = useSpriteInfo(scene.paths.vtt ?? undefined);
   const [spriteItems, setSpriteItems] = useState<ISceneSpriteItem[]>();
 
   useEffect(() => {
-    if (!scene.paths.vtt) return;
+    if (!spriteInfo) return;
-    fetchSpriteInfo(scene.paths.vtt).then((sprites) => {
+    let totalWidth = 0;
-      if (!sprites) return;
+    const newSprites = spriteInfo?.map((sprite, index) => {
-      let totalWidth = 0;
+      totalWidth += sprite.w;
-      const newSprites = sprites?.map((sprite, index) => {
+      const left = sprite.w * index;
-        totalWidth += sprite.w;
+      const style = {
-        const left = sprite.w * index;
+        width: `${sprite.w}px`,
-        const style = {
+        height: `${sprite.h}px`,
-          width: `${sprite.w}px`,
+        backgroundPosition: `${-sprite.x}px ${-sprite.y}px`,
-          height: `${sprite.h}px`,
+        backgroundImage: `url(${sprite.url})`,
-          backgroundPosition: `${-sprite.x}px ${-sprite.y}px`,
+        left: `${left}px`,
-          backgroundImage: `url(${sprite.url})`,
+      };
-          left: `${left}px`,
+      const start = TextUtils.secondsToTimestamp(sprite.start);
-        };
+      const end = TextUtils.secondsToTimestamp(sprite.end);
-        const start = TextUtils.secondsToTimestamp(sprite.start);
+      return {
-        const end = TextUtils.secondsToTimestamp(sprite.end);
+        style,
-        return {
+        time: `${start} - ${end}`,
-          style,
+      };
-          time: `${start} - ${end}`,
-        };
-      });
-      setScrubWidth(totalWidth);
-      setSpriteItems(newSprites);
     });
-  }, [scene]);
+    setScrubWidth(totalWidth);
+    setSpriteItems(newSprites);
+  }, [spriteInfo]);
 
   useEffect(() => {
     const onResize = (entries: ResizeObserverEntry[]) => {

ui/v2.5/src/components/Scenes/PreviewScrubber.tsx

@@ -0,0 +1,173 @@
+import React, { useMemo } from "react";
+import { useDebounce } from "src/hooks/debounce";
+import { useSpriteInfo } from "src/hooks/sprite";
+import TextUtils from "src/utils/text";
+
+interface IHoverScrubber {
+  totalSprites: number;
+  activeIndex: number | undefined;
+  setActiveIndex: (index: number | undefined) => void;
+  onClick?: (index: number) => void;
+}
+
+const HoverScrubber: React.FC<IHoverScrubber> = ({
+  totalSprites,
+  activeIndex,
+  setActiveIndex,
+  onClick,
+}) => {
+  function getActiveIndex(e: React.MouseEvent<HTMLDivElement, MouseEvent>) {
+    const { width } = e.currentTarget.getBoundingClientRect();
+    const x = e.nativeEvent.offsetX;
+
+    return Math.floor((x / width) * totalSprites);
+  }
+
+  function onMouseMove(e: React.MouseEvent<HTMLDivElement, MouseEvent>) {
+    const relatedTarget = e.currentTarget;
+
+    if (relatedTarget !== e.target) return;
+
+    setActiveIndex(getActiveIndex(e));
+  }
+
+  function onMouseLeave() {
+    setActiveIndex(undefined);
+  }
+
+  function onScrubberClick(e: React.MouseEvent<HTMLDivElement, MouseEvent>) {
+    if (!onClick) return;
+
+    const relatedTarget = e.currentTarget;
+
+    if (relatedTarget !== e.target) return;
+
+    e.preventDefault();
+    onClick(getActiveIndex(e));
+  }
+
+  const indicatorStyle = useMemo(() => {
+    if (activeIndex === undefined) return {};
+
+    const width = (activeIndex / totalSprites) * 100;
+
+    return {
+      width: `${width}%`,
+    };
+  }, [activeIndex, totalSprites]);
+
+  return (
+    <div className="hover-scrubber">
+      <div
+        className="hover-scrubber-area"
+        onMouseMove={onMouseMove}
+        onMouseLeave={onMouseLeave}
+        onClick={onScrubberClick}
+      />
+      <div className="hover-scrubber-indicator">
+        {activeIndex !== undefined && (
+          <div
+            className="hover-scrubber-indicator-marker"
+            style={indicatorStyle}
+          ></div>
+        )}
+      </div>
+    </div>
+  );
+};
+
+interface IScenePreviewProps {
+  vttPath: string | undefined;
+  onClick?: (timestamp: number) => void;
+}
+
+function scaleToFit(dimensions: { w: number; h: number }, bounds: DOMRect) {
+  const rw = bounds.width / dimensions.w;
+  const rh = bounds.height / dimensions.h;
+
+  // for consistency, use max by default and min for portrait
+  if (dimensions.w > dimensions.h) {
+    return Math.max(rw, rh);
+  }
+
+  return Math.min(rw, rh);
+}
+
+export const PreviewScrubber: React.FC<IScenePreviewProps> = ({
+  vttPath,
+  onClick,
+}) => {
+  const imageParentRef = React.useRef<HTMLDivElement>(null);
+
+  const [activeIndex, setActiveIndex] = React.useState<number | undefined>();
+
+  const debounceSetActiveIndex = useDebounce(
+    setActiveIndex,
+    [setActiveIndex],
+    10
+  );
+
+  const spriteInfo = useSpriteInfo(vttPath);
+
+  const style = useMemo(() => {
+    if (!spriteInfo || activeIndex === undefined || !imageParentRef.current) {
+      return {};
+    }
+
+    const sprite = spriteInfo[activeIndex];
+
+    const clientRect = imageParentRef.current?.getBoundingClientRect();
+    const scale = clientRect ? scaleToFit(sprite, clientRect) : 1;
+
+    return {
+      backgroundPosition: `${-sprite.x}px ${-sprite.y}px`,
+      backgroundImage: `url(${sprite.url})`,
+      width: `${sprite.w}px`,
+      height: `${sprite.h}px`,
+      transform: `scale(${scale})`,
+    };
+  }, [spriteInfo, activeIndex, imageParentRef]);
+
+  const currentTime = useMemo(() => {
+    if (!spriteInfo || activeIndex === undefined) {
+      return undefined;
+    }
+
+    const sprite = spriteInfo[activeIndex];
+
+    const start = TextUtils.secondsToTimestamp(sprite.start);
+
+    return start;
+  }, [activeIndex, spriteInfo]);
+
+  function onScrubberClick(index: number) {
+    if (!spriteInfo || !onClick) {
+      return;
+    }
+
+    const sprite = spriteInfo[index];
+
+    onClick(sprite.start);
+  }
+
+  if (!spriteInfo) return null;
+
+  return (
+    <div className="preview-scrubber">
+      {activeIndex !== undefined && spriteInfo && (
+        <div className="scene-card-preview-image" ref={imageParentRef}>
+          <div className="scrubber-image" style={style}></div>
+          {currentTime !== undefined && (
+            <div className="scrubber-timestamp">{currentTime}</div>
+          )}
+        </div>
+      )}
+      <HoverScrubber
+        totalSprites={81}
+        activeIndex={activeIndex}
+        setActiveIndex={(i) => debounceSetActiveIndex(i)}
+        onClick={onScrubberClick}
+      />
+    </div>
+  );
+};

ui/v2.5/src/components/Scenes/SceneCard.tsx

@@ -1,6 +1,6 @@
 import React, { useEffect, useMemo, useRef } from "react";
 import { Button, ButtonGroup } from "react-bootstrap";
-import { Link } from "react-router-dom";
+import { Link, useHistory } from "react-router-dom";
 import cx from "classnames";
 import * as GQL from "src/core/generated-graphql";
 import { Icon } from "../Shared/Icon";
@@ -25,12 +25,15 @@ import {
   faTag,
 } from "@fortawesome/free-solid-svg-icons";
 import { objectPath, objectTitle } from "src/core/files";
+import { PreviewScrubber } from "./PreviewScrubber";
 
 interface IScenePreviewProps {
   isPortrait: boolean;
   image?: string;
   video?: string;
   soundActive: boolean;
+  vttPath?: string;
+  onScrubberClick?: (timestamp: number) => void;
 }
 
 export const ScenePreview: React.FC<IScenePreviewProps> = ({
@@ -38,6 +41,8 @@ export const ScenePreview: React.FC<IScenePreviewProps> = ({
   video,
   isPortrait,
   soundActive,
+  vttPath,
+  onScrubberClick,
 }) => {
   const videoEl = useRef<HTMLVideoElement>(null);
 
@@ -72,6 +77,7 @@ export const ScenePreview: React.FC<IScenePreviewProps> = ({
         ref={videoEl}
         src={video}
       />
+      <PreviewScrubber vttPath={vttPath} onClick={onScrubberClick} />
     </div>
   );
 };
@@ -90,6 +96,7 @@ interface ISceneCardProps {
 export const SceneCard: React.FC<ISceneCardProps> = (
   props: ISceneCardProps
 ) => {
+  const history = useHistory();
   const { configuration } = React.useContext(ConfigurationContext);
 
   const file = useMemo(
@@ -383,6 +390,18 @@ export const SceneCard: React.FC<ISceneCardProps> = (
       })
     : `/scenes/${props.scene.id}`;
 
+  function onScrubberClick(timestamp: number) {
+    const link = props.queue
+      ? props.queue.makeLink(props.scene.id, {
+          sceneIndex: props.index,
+          continue: cont,
+          start: timestamp,
+        })
+      : `/scenes/${props.scene.id}?t=${timestamp}`;
+
+    history.push(link);
+  }
+
   return (
     <GridCard
       className={`scene-card ${zoomIndex()} ${filelessClass()}`}
@@ -404,6 +423,8 @@ export const SceneCard: React.FC<ISceneCardProps> = (
             video={props.scene.paths.preview ?? undefined}
             isPortrait={isPortrait()}
             soundActive={configuration?.interface?.soundOnPreview ?? false}
+            vttPath={props.scene.paths.vtt ?? undefined}
+            onScrubberClick={onScrubberClick}
           />
           <RatingBanner rating={props.scene.rating100} />
           {maybeRenderSceneSpecsOverlay()}

ui/v2.5/src/components/Scenes/styles.scss

@@ -643,3 +643,66 @@ input[type="range"].blue-slider {
 .scrape-dialog .rating-number.disabled {
   padding-left: 0.5em;
 }
+
+.preview-scrubber {
+  height: 100%;
+  position: absolute;
+  width: 100%;
+
+  .scene-card-preview-image {
+    align-items: center;
+    display: flex;
+    justify-content: center;
+    overflow: hidden;
+  }
+
+  .scrubber-image {
+    height: 100%;
+    width: 100%;
+  }
+
+  .scrubber-timestamp {
+    bottom: calc(20px + 0.25rem);
+    font-weight: 400;
+    opacity: 0.75;
+    position: absolute;
+    right: 0.7rem;
+    text-shadow: 0 0 3px #000;
+  }
+}
+
+.hover-scrubber {
+  bottom: 0;
+  height: 20px;
+  overflow: hidden;
+  position: absolute;
+  width: 100%;
+
+  .hover-scrubber-area {
+    cursor: col-resize;
+    height: 100%;
+    position: absolute;
+    width: 100%;
+    z-index: 1;
+  }
+
+  .hover-scrubber-indicator {
+    background-color: rgba(255, 255, 255, 0.1);
+    bottom: -100%;
+    height: 100%;
+    position: absolute;
+    transition: bottom 0.2s ease-in-out;
+    width: 100%;
+
+    .hover-scrubber-indicator-marker {
+      background-color: rgba(255, 0, 0, 0.5);
+      bottom: 0;
+      height: 5px;
+      position: absolute;
+    }
+  }
+
+  &:hover .hover-scrubber-indicator {
+    bottom: 0;
+  }
+}

ui/v2.5/src/hooks/sprite.ts

@@ -0,0 +1,62 @@
+import { useEffect, useState } from "react";
+import { WebVTT } from "videojs-vtt.js";
+
+export interface ISceneSpriteInfo {
+  url: string;
+  start: number;
+  end: number;
+  x: number;
+  y: number;
+  w: number;
+  h: number;
+}
+
+function getSpriteInfo(vttPath: string, response: string) {
+  const sprites: ISceneSpriteInfo[] = [];
+
+  const parser = new WebVTT.Parser(window, WebVTT.StringDecoder());
+  parser.oncue = (cue: VTTCue) => {
+    const match = cue.text.match(/^([^#]*)#xywh=(\d+),(\d+),(\d+),(\d+)$/i);
+    if (!match) return;
+
+    sprites.push({
+      url: new URL(match[1], vttPath).href,
+      start: cue.startTime,
+      end: cue.endTime,
+      x: Number(match[2]),
+      y: Number(match[3]),
+      w: Number(match[4]),
+      h: Number(match[5]),
+    });
+  };
+  parser.parse(response);
+  parser.flush();
+
+  return sprites;
+}
+
+export function useSpriteInfo(vttPath: string | undefined) {
+  const [spriteInfo, setSpriteInfo] = useState<
+    ISceneSpriteInfo[] | undefined
+  >();
+
+  useEffect(() => {
+    if (!vttPath) {
+      setSpriteInfo(undefined);
+      return;
+    }
+
+    fetch(vttPath).then((response) => {
+      if (!response.ok) {
+        setSpriteInfo(undefined);
+        return;
+      }
+
+      response.text().then((text) => {
+        setSpriteInfo(getSpriteInfo(vttPath, text));
+      });
+    });
+  }, [vttPath]);
+
+  return spriteInfo;
+}

ui/v2.5/src/models/sceneQueue.ts

@@ -9,6 +9,7 @@ export interface IPlaySceneOptions {
   newPage?: number;
   autoPlay?: boolean;
   continue?: boolean;
+  start?: number;
 }
 
 export class SceneQueue {
@@ -117,6 +118,9 @@ export class SceneQueue {
     if (options.continue !== undefined) {
       params.push("continue=" + options.continue);
     }
+    if (options.start !== undefined) {
+      params.push("t=" + options.start);
+    }
     return `/scenes/${sceneID}${params.length ? "?" + params.join("&") : ""}`;
   }
 }