Thumbnails in backend

src/main/ThumbnailGeneration.tsx

@@ -22,6 +22,7 @@ import {
   setJumpTriggered,
   setAspectRatio,
   selectPrimaryThumbnailTrack,
+  setThumbnailTime,
 } from "../redux/videoSlice";
 import { Track } from "../types";
 import Timeline from "./Timeline";
@@ -210,8 +211,11 @@ const ThumbnailActions: React.FC<{
   //   *track: Generate to
   //   *index: Generate from
   const generate = (track: Track, index: number) => {
+    const time = generateRefs.current[index]?.getCurrentTime();
+    const timeObject = time ? { time: time.toString(), flavorType: track.flavor.type } : undefined;
     const uri = generateRefs.current[index]?.captureVideo();
     dispatch(setThumbnail({ id: track.id, uri: uri }));
+    dispatch(setThumbnailTime({ id: track.id, time: timeObject }));
     dispatch(setHasChanges(true));
   };
 

src/main/ThumbnailSelect.tsx

@@ -1,7 +1,7 @@
 import { css, SerializedStyles } from "@emotion/react";
 import { IconType } from "react-icons";
 import { LuCamera, LuCopy, LuCircleX, LuUpload } from "react-icons/lu";
-import React from "react";
+import React, { useEffect, useRef, useState } from "react";
 import { useTranslation } from "react-i18next";
 import { useAppDispatch, useAppSelector } from "../redux/store";
 import {
@@ -16,11 +16,13 @@ import {
   setHasChanges,
   setThumbnail,
   setThumbnails,
+  setThumbnailTime,
 } from "../redux/videoSlice";
 import { Track } from "../types";
 import { ThemedTooltip } from "./Tooltip";
 import { ProtoButton } from "@opencast/appkit";
 import { setIndex, setIsDisplayEditView } from "../redux/thumbnailSlice";
+import ReactPlayer from "react-player";
 
 /**
  * Choose between various thumbnail actions for the available tracks.
@@ -98,6 +100,7 @@ const ThumbnailSelector: React.FC<{
         track={track}
         trackIndex={trackIndex}
       />
+      <WorkaroundThumbnailGenerator track={track} />
     </div>
   );
 };
@@ -265,6 +268,7 @@ export const UploadButton: React.FC<{
       if (e.target && e.target.result) {
         const uri = e.target.result as string; // We know this must be string because we use "readAsDataURL"
         dispatch(setThumbnail({ id: track.id, uri: uri }));
+        dispatch(setThumbnailTime({ id: track.id, time: undefined }));
         dispatch(setHasChanges(true));
       }
     };
@@ -311,8 +315,13 @@ export const DiscardButton: React.FC<{
 
   const originalThumbnails = useAppSelector(selectOriginalThumbnails);
 
+  const originalThumbnail = originalThumbnails.find(e => e.id === track.id);
+  const active = (track.thumbnailUri && track.thumbnailUri.startsWith("data") && !track.thumbnailTime)
+    || (track.thumbnailTime && originalThumbnail && track.thumbnailTime != originalThumbnail.time);
+
   const discardThumbnail = (id: string) => {
-    dispatch(setThumbnail({ id: id, uri: originalThumbnails.find(e => e.id === id)?.uri }));
+    dispatch(setThumbnail({ id: id, uri: originalThumbnail?.uri }));
+    dispatch(setThumbnailTime({ id: id, time: originalThumbnail?.time }));
   };
 
   return (
@@ -322,7 +331,7 @@ export const DiscardButton: React.FC<{
       tooltipText={t("thumbnail.buttonDiscard-tooltip")}
       ariaLabel={t("thumbnail.buttonDiscard-tooltip-aria")}
       Icon={LuCircleX}
-      active={(track.thumbnailUri && track.thumbnailUri.startsWith("data") ? true : false)}
+      active={(active ? true : false)}
       index={index}
       overwriteCSS={overwriteCSS}
     />
@@ -342,21 +351,21 @@ export const UseForAllTracksButton: React.FC<{
   const tracks = useAppSelector(selectTracks);
 
   // Set the given thumbnail for all tracks
-  const setForOtherThumbnails = (uri: string | undefined) => {
-    if (uri === undefined) {
-      return;
-    }
+  const setForOtherThumbnails = ({ uri, time }: {
+    uri: string | undefined,
+    time?: { time: string; flavorType: string }
+  }) => {
     const thumbnails = [];
     for (const track of tracks) {
-      thumbnails.push({ id: track.id, uri: uri });
+      thumbnails.push({ id: track.id, uri, time });
     }
     dispatch(setThumbnails(thumbnails));
     dispatch(setHasChanges(true));
   };
 
   return (
     <ThumbnailButton
-      handler={() => { setForOtherThumbnails(track.thumbnailUri); }}
+      handler={() => { setForOtherThumbnails({ uri: track.thumbnailUri, time: track.thumbnailTime }); }}
       text={t("thumbnail.buttonUseForOtherThumbnails")}
       tooltipText={t("thumbnail.buttonUseForOtherThumbnails-tooltip")}
       ariaLabel={t("thumbnail.buttonUseForOtherThumbnails-tooltip-aria")}
@@ -454,6 +463,74 @@ export const ThumbnailButton: React.FC<{
   );
 };
 
+/**
+ * Generates a temporary thumbnail from a timestamp
+ *
+ * Workaround for the backend being unable to send us thumbnails from
+ * publications. This way, we can at least show a thumbnail to a user
+ * if they previously generated one via timestamp.
+ */
+const WorkaroundThumbnailGenerator: React.FC<{
+  track: Track,
+}> = ({ track }) => {
+  const dispatch = useAppDispatch();
+
+  const tracks = useAppSelector(selectTracks);
+  const thumbnailTime = track.thumbnailTime;
+  const thumbnailTrack = thumbnailTime
+    ? tracks.find(t => t.flavor.type === thumbnailTime.flavorType)
+    : undefined;
+
+  const ref = useRef<ReactPlayer>(null);
+  const [ready, setReady] = useState(false);
+  const [seeked, setSeeked] = useState(false);
+
+  useEffect(() => {
+    if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri) {
+      ref.current.seekTo(parseFloat(track.thumbnailTime.time), "seconds");
+    }
+  }, [dispatch, ready, track]);
+
+  useEffect(() => {
+    if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri
+      && seeked) {
+      const videoElement = ref.current?.getInternalPlayer() as HTMLVideoElement;
+      const canvas = document.createElement("canvas");
+      canvas.width = videoElement.videoWidth;
+      canvas.height = videoElement.videoHeight;
+      const canvasContext = canvas.getContext("2d");
+      if (canvasContext !== null) {
+        canvasContext.drawImage(videoElement, 0, 0);
+        const uri = canvas.toDataURL("image/png");
+
+        if (uri) {
+          dispatch(setThumbnail({ id: track.id, uri: uri }));
+        }
+      }
+    }
+  }, [dispatch, ready, seeked, track]);
+
+  const playerStyle = css({
+    display: "none",
+  });
+
+  if (!thumbnailTrack) {
+    return null;
+  }
+
+  return (
+    <ReactPlayer url={thumbnailTrack.uri}
+      css={playerStyle}
+      ref={ref}
+      width="unset"
+      height="100%"
+      playing={false}
+      onReady={() => setReady(true)}
+      onSeek={() => setSeeked(true)}
+    />
+  );
+};
+
 /**
  * Shared CSS
  */

src/main/VideoPlayers.tsx

@@ -116,6 +116,7 @@ const VideoPlayers: React.FC<{
 export interface VideoPlayerForwardRef {
   captureVideo: () => string | undefined,
   getWidth: () => number,
+  getCurrentTime: () => number,
 }
 
 interface VideoPlayerProps {
@@ -390,6 +391,9 @@ export const VideoPlayer = React.forwardRef<VideoPlayerForwardRef, VideoPlayerPr
       getWidth() {
         return (ref.current?.getInternalPlayer() as HTMLVideoElement).clientWidth;
       },
+      getCurrentTime() {
+        return (ref.current?.getInternalPlayer() as HTMLVideoElement).currentTime;
+      },
     }));
 
     const reactPlayerStyle = css({

src/redux/videoSlice.ts

@@ -27,7 +27,7 @@ export interface video {
   hasChanges: boolean,            // Did user make changes in cutting view since last save
   timelineZoom: number,           // Zoom multiplicator for the timeline,
   waveformImages: string[];
-  originalThumbnails: { id: Track["id"], uri: Track["thumbnailUri"]; }[];
+  originalThumbnails: { id: Track["id"], uri: Track["thumbnailUri"], time: Track["thumbnailTime"]; }[];
 
   videoURLs: string[],  // Links to each video
   videoCount: number,   // Total number of videos
@@ -233,11 +233,19 @@ const videoSlice = createSlice({
     setThumbnail: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"]; }>) => {
       setThumbnailHelper(state, action.payload.id, action.payload.uri);
     },
-    setThumbnails: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"]; }[]>) => {
+    setThumbnails: (state, action: PayloadAction<{
+      id: Track["id"],
+      uri: Track["thumbnailUri"],
+      time: Track["thumbnailTime"];
+    }[]>) => {
       for (const element of action.payload) {
         setThumbnailHelper(state, element.id, element.uri);
+        setThumbnailTimeHelper(state, element.id, element.time);
       }
     },
+    setThumbnailTime: (state, action: PayloadAction<{ id: Track["id"], time: Track["thumbnailTime"]; }>) => {
+      setThumbnailTimeHelper(state, action.payload.id, action.payload.time);
+    },
     removeThumbnail: (state, action: PayloadAction<string>) => {
       const index = state.tracks.findIndex(t => t.id === action.payload);
       state.tracks[index].thumbnailUri = undefined;
@@ -377,7 +385,7 @@ const videoSlice = createSlice({
           return waveformURI;
         }) : state.waveformImages;
         state.originalThumbnails = state.tracks.map(
-          (track: Track) => { return { id: track.id, uri: track.thumbnailUri }; },
+          (track: Track) => { return { id: track.id, uri: track.thumbnailUri, time: track.thumbnailTime }; },
         );
 
         state.lockingActive = payload.locking_active;
@@ -573,6 +581,13 @@ const setThumbnailHelper = (state: video, id: Track["id"], uri: Track["thumbnail
   }
 };
 
+const setThumbnailTimeHelper = (state: video, id: Track["id"], time: Track["thumbnailTime"]) => {
+  const index = state.tracks.findIndex(t => t.id === id);
+  if (index >= 0) {
+    state.tracks[index].thumbnailTime = time;
+  }
+};
+
 export const {
   addSegment,
   cut,
@@ -603,6 +618,7 @@ export const {
   setSelectedWorkflowIndex,
   setThumbnail,
   setThumbnails,
+  setThumbnailTime,
   setVideoEnabled,
   setVolume,
   setWaveformImages,

src/types.ts

@@ -16,6 +16,10 @@ export interface Track {
   video_stream: {available: boolean, enabled: boolean, thumbnail_uri: string},
   thumbnailUri: string | undefined,
   thumbnailPriority: number,
+  thumbnailTime?: {
+    time: string,
+    flavorType: string,
+  },
 }
 
 export interface Flavor {