From 85d27dad48d013d5ebce185e64c0226f6e7da49f Mon Sep 17 00:00:00 2001 From: Arnei Date: Wed, 3 Jun 2026 17:02:12 +0200 Subject: [PATCH 1/4] Thumbnails in backend Currently when generating a thumbnail, we create it here in the frontend and then send it to the backend. With this patch, we instead send the timestamp it was generated from to the backend. The goal is to let the backend generate the actual thumbnail, instead of the frontend. This avoids quality issues when generating thumbnails from low resolution videos in the frontend. It also allows for respecting institituion specific thumbnail settings (e.g. encoding profiles). The downside is that this relies on workflow properties. As such, admins will have to adapt their workflows or thumbnail generation will be broken for them. This change should not impact the user experience of using the editor frontend. --- src/main/ThumbnailGeneration.tsx | 3 ++ src/main/ThumbnailSelect.tsx | 63 +++++++++++++++++++++++++++++++- src/main/VideoPlayers.tsx | 4 ++ src/redux/videoSlice.ts | 11 ++++++ src/types.ts | 1 + 5 files changed, 81 insertions(+), 1 deletion(-) diff --git a/src/main/ThumbnailGeneration.tsx b/src/main/ThumbnailGeneration.tsx index 3a31768c8..33abd4bae 100644 --- a/src/main/ThumbnailGeneration.tsx +++ b/src/main/ThumbnailGeneration.tsx @@ -22,6 +22,7 @@ import { setJumpTriggered, setAspectRatio, selectPrimaryThumbnailTrack, + setThumbnailTime, } from "../redux/videoSlice"; import { Track } from "../types"; import Timeline from "./Timeline"; @@ -210,8 +211,10 @@ const ThumbnailActions: React.FC<{ // *track: Generate to // *index: Generate from const generate = (track: Track, index: number) => { + const time = generateRefs.current[index]?.getCurrentTime(); const uri = generateRefs.current[index]?.captureVideo(); dispatch(setThumbnail({ id: track.id, uri: uri })); + dispatch(setThumbnailTime({ id: track.id, time: time?.toString() })); dispatch(setHasChanges(true)); }; diff --git a/src/main/ThumbnailSelect.tsx b/src/main/ThumbnailSelect.tsx index 857a927fd..fbecbec89 100644 --- a/src/main/ThumbnailSelect.tsx +++ b/src/main/ThumbnailSelect.tsx @@ -1,7 +1,7 @@ import { css, SerializedStyles } from "@emotion/react"; import { IconType } from "react-icons"; import { LuCamera, LuCopy, LuCircleX, LuUpload } from "react-icons/lu"; -import React from "react"; +import React, { useEffect, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { useAppDispatch, useAppSelector } from "../redux/store"; import { @@ -16,11 +16,13 @@ import { setHasChanges, setThumbnail, setThumbnails, + setThumbnailTime, } from "../redux/videoSlice"; import { Track } from "../types"; import { ThemedTooltip } from "./Tooltip"; import { ProtoButton } from "@opencast/appkit"; import { setIndex, setIsDisplayEditView } from "../redux/thumbnailSlice"; +import ReactPlayer from "react-player"; /** * Choose between various thumbnail actions for the available tracks. @@ -98,6 +100,7 @@ const ThumbnailSelector: React.FC<{ track={track} trackIndex={trackIndex} /> + ); }; @@ -265,6 +268,7 @@ export const UploadButton: React.FC<{ if (e.target && e.target.result) { const uri = e.target.result as string; // We know this must be string because we use "readAsDataURL" dispatch(setThumbnail({ id: track.id, uri: uri })); + dispatch(setThumbnailTime({ id: track.id, time: undefined })); dispatch(setHasChanges(true)); } }; @@ -454,6 +458,63 @@ export const ThumbnailButton: React.FC<{ ); }; +/** + * Generates a temporary thumbnail from a timestamp + * + * Workaround for the backend being unable to send us thumbnails from + * publications. This way, we can at least show a thumbnail to a user + * if they previously generated one via timestamp. + */ +const WorkaroundThumbnailGenerator: React.FC<{ + track: Track, +}> = ({ track }) => { + const dispatch = useAppDispatch(); + + const ref = useRef(null); + const [ready, setReady] = useState(false); + const [seeked, setSeeked] = useState(false); + + useEffect(() => { + if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri) { + ref.current.seekTo(parseFloat(track.thumbnailTime), "seconds"); + } + }, [dispatch, ready, track]); + + useEffect(() => { + if (seeked) { + const videoElement = ref.current?.getInternalPlayer() as HTMLVideoElement; + const canvas = document.createElement("canvas"); + canvas.width = videoElement.videoWidth; + canvas.height = videoElement.videoHeight; + const canvasContext = canvas.getContext("2d"); + if (canvasContext !== null) { + canvasContext.drawImage(videoElement, 0, 0); + const uri = canvas.toDataURL("image/png"); + + if (uri) { + dispatch(setThumbnail({ id: track.id, uri: uri })); + } + } + } + }, [dispatch, seeked, track]); + + const playerStyle = css({ + display: "none", + }); + + return ( + setReady(true)} + onSeek={() => setSeeked(true)} + /> + ); +}; + /** * Shared CSS */ diff --git a/src/main/VideoPlayers.tsx b/src/main/VideoPlayers.tsx index d5eee634e..83ec2bba6 100644 --- a/src/main/VideoPlayers.tsx +++ b/src/main/VideoPlayers.tsx @@ -116,6 +116,7 @@ const VideoPlayers: React.FC<{ export interface VideoPlayerForwardRef { captureVideo: () => string | undefined, getWidth: () => number, + getCurrentTime: () => number, } interface VideoPlayerProps { @@ -390,6 +391,9 @@ export const VideoPlayer = React.forwardRef) => { + setThumbnailTimeHelper(state, action.payload.id, action.payload.time); + }, removeThumbnail: (state, action: PayloadAction) => { const index = state.tracks.findIndex(t => t.id === action.payload); state.tracks[index].thumbnailUri = undefined; @@ -573,6 +576,13 @@ const setThumbnailHelper = (state: video, id: Track["id"], uri: Track["thumbnail } }; +const setThumbnailTimeHelper = (state: video, id: Track["id"], time: Track["thumbnailTime"]) => { + const index = state.tracks.findIndex(t => t.id === id); + if (index >= 0) { + state.tracks[index].thumbnailTime = time; + } +}; + export const { addSegment, cut, @@ -603,6 +613,7 @@ export const { setSelectedWorkflowIndex, setThumbnail, setThumbnails, + setThumbnailTime, setVideoEnabled, setVolume, setWaveformImages, diff --git a/src/types.ts b/src/types.ts index ef3f9fa7c..1230dc4e2 100644 --- a/src/types.ts +++ b/src/types.ts @@ -16,6 +16,7 @@ export interface Track { video_stream: {available: boolean, enabled: boolean, thumbnail_uri: string}, thumbnailUri: string | undefined, thumbnailPriority: number, + thumbnailTime?: string, } export interface Flavor { From 20928b4608dd25f2c28d079ae30b525d1a5f85cd Mon Sep 17 00:00:00 2001 From: Arnei Date: Tue, 16 Jun 2026 10:26:37 +0200 Subject: [PATCH 2/4] Fix useForAllTrack button in thumbnails Was not working for timestamp thumbnails, because I forgot to adapt the code for that --- src/main/ThumbnailSelect.tsx | 9 +++------ src/redux/videoSlice.ts | 7 ++++++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/ThumbnailSelect.tsx b/src/main/ThumbnailSelect.tsx index fbecbec89..358d5fc87 100644 --- a/src/main/ThumbnailSelect.tsx +++ b/src/main/ThumbnailSelect.tsx @@ -346,13 +346,10 @@ export const UseForAllTracksButton: React.FC<{ const tracks = useAppSelector(selectTracks); // Set the given thumbnail for all tracks - const setForOtherThumbnails = (uri: string | undefined) => { - if (uri === undefined) { - return; - } + const setForOtherThumbnails = ({ uri, time }: {uri: string | undefined, time: string | undefined}) => { const thumbnails = []; for (const track of tracks) { - thumbnails.push({ id: track.id, uri: uri }); + thumbnails.push({ id: track.id, uri, time }); } dispatch(setThumbnails(thumbnails)); dispatch(setHasChanges(true)); @@ -360,7 +357,7 @@ export const UseForAllTracksButton: React.FC<{ return ( { setForOtherThumbnails(track.thumbnailUri); }} + handler={() => { setForOtherThumbnails({uri: track.thumbnailUri, time: track.thumbnailTime }); }} text={t("thumbnail.buttonUseForOtherThumbnails")} tooltipText={t("thumbnail.buttonUseForOtherThumbnails-tooltip")} ariaLabel={t("thumbnail.buttonUseForOtherThumbnails-tooltip-aria")} diff --git a/src/redux/videoSlice.ts b/src/redux/videoSlice.ts index 31d18dee4..715cb2434 100644 --- a/src/redux/videoSlice.ts +++ b/src/redux/videoSlice.ts @@ -233,9 +233,14 @@ const videoSlice = createSlice({ setThumbnail: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"]; }>) => { setThumbnailHelper(state, action.payload.id, action.payload.uri); }, - setThumbnails: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"]; }[]>) => { + setThumbnails: (state, action: PayloadAction<{ + id: Track["id"], + uri: Track["thumbnailUri"], + time: Track["thumbnailTime"]; + }[]>) => { for (const element of action.payload) { setThumbnailHelper(state, element.id, element.uri); + setThumbnailTimeHelper(state, element.id, element.time); } }, setThumbnailTime: (state, action: PayloadAction<{ id: Track["id"], time: Track["thumbnailTime"]; }>) => { From 885fef3a4bdf76b4ad1a79cfb9fb9cca6114b1ff Mon Sep 17 00:00:00 2001 From: Arnei Date: Tue, 16 Jun 2026 11:45:06 +0200 Subject: [PATCH 3/4] Fix discard button not enabled when it should be --- src/main/ThumbnailSelect.tsx | 16 +++++++++++----- src/redux/videoSlice.ts | 6 +++--- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/src/main/ThumbnailSelect.tsx b/src/main/ThumbnailSelect.tsx index 358d5fc87..0bb1dec9f 100644 --- a/src/main/ThumbnailSelect.tsx +++ b/src/main/ThumbnailSelect.tsx @@ -315,8 +315,13 @@ export const DiscardButton: React.FC<{ const originalThumbnails = useAppSelector(selectOriginalThumbnails); + const originalThumbnail = originalThumbnails.find(e => e.id === track.id); + const active = (track.thumbnailUri && track.thumbnailUri.startsWith("data") && !track.thumbnailTime) + || (track.thumbnailTime && originalThumbnail && track.thumbnailTime != originalThumbnail.time); + const discardThumbnail = (id: string) => { - dispatch(setThumbnail({ id: id, uri: originalThumbnails.find(e => e.id === id)?.uri })); + dispatch(setThumbnail({ id: id, uri: originalThumbnail?.uri })); + dispatch(setThumbnailTime({ id: id, time: originalThumbnail?.time })); }; return ( @@ -326,7 +331,7 @@ export const DiscardButton: React.FC<{ tooltipText={t("thumbnail.buttonDiscard-tooltip")} ariaLabel={t("thumbnail.buttonDiscard-tooltip-aria")} Icon={LuCircleX} - active={(track.thumbnailUri && track.thumbnailUri.startsWith("data") ? true : false)} + active={(active ? true : false)} index={index} overwriteCSS={overwriteCSS} /> @@ -357,7 +362,7 @@ export const UseForAllTracksButton: React.FC<{ return ( { setForOtherThumbnails({uri: track.thumbnailUri, time: track.thumbnailTime }); }} + handler={() => { setForOtherThumbnails({ uri: track.thumbnailUri, time: track.thumbnailTime }); }} text={t("thumbnail.buttonUseForOtherThumbnails")} tooltipText={t("thumbnail.buttonUseForOtherThumbnails-tooltip")} ariaLabel={t("thumbnail.buttonUseForOtherThumbnails-tooltip-aria")} @@ -478,7 +483,8 @@ const WorkaroundThumbnailGenerator: React.FC<{ }, [dispatch, ready, track]); useEffect(() => { - if (seeked) { + if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri + && seeked) { const videoElement = ref.current?.getInternalPlayer() as HTMLVideoElement; const canvas = document.createElement("canvas"); canvas.width = videoElement.videoWidth; @@ -493,7 +499,7 @@ const WorkaroundThumbnailGenerator: React.FC<{ } } } - }, [dispatch, seeked, track]); + }, [dispatch, ready, seeked, track]); const playerStyle = css({ display: "none", diff --git a/src/redux/videoSlice.ts b/src/redux/videoSlice.ts index 715cb2434..b846f0a5b 100644 --- a/src/redux/videoSlice.ts +++ b/src/redux/videoSlice.ts @@ -27,7 +27,7 @@ export interface video { hasChanges: boolean, // Did user make changes in cutting view since last save timelineZoom: number, // Zoom multiplicator for the timeline, waveformImages: string[]; - originalThumbnails: { id: Track["id"], uri: Track["thumbnailUri"]; }[]; + originalThumbnails: { id: Track["id"], uri: Track["thumbnailUri"], time: Track["thumbnailTime"]; }[]; videoURLs: string[], // Links to each video videoCount: number, // Total number of videos @@ -233,7 +233,7 @@ const videoSlice = createSlice({ setThumbnail: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"]; }>) => { setThumbnailHelper(state, action.payload.id, action.payload.uri); }, - setThumbnails: (state, action: PayloadAction<{ + setThumbnails: (state, action: PayloadAction<{ id: Track["id"], uri: Track["thumbnailUri"], time: Track["thumbnailTime"]; @@ -385,7 +385,7 @@ const videoSlice = createSlice({ return waveformURI; }) : state.waveformImages; state.originalThumbnails = state.tracks.map( - (track: Track) => { return { id: track.id, uri: track.thumbnailUri }; }, + (track: Track) => { return { id: track.id, uri: track.thumbnailUri, time: track.thumbnailTime }; }, ); state.lockingActive = payload.locking_active; From 83691baa36a03fe9c83a75b2a185b94f0c1caac0 Mon Sep 17 00:00:00 2001 From: Arnei Date: Fri, 19 Jun 2026 15:37:30 +0200 Subject: [PATCH 4/4] Add flavor to thumbnail time for useAllTracks Not only sends the timestamp a thumbnail should be generated from, but also the flavor type of the video it should be generated from. This is necessary when "useWithAllTracks" is used, because it adds a thumbnail from one video to other videos. --- src/main/ThumbnailGeneration.tsx | 3 ++- src/main/ThumbnailSelect.tsx | 19 ++++++++++++++++--- src/types.ts | 5 ++++- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/ThumbnailGeneration.tsx b/src/main/ThumbnailGeneration.tsx index 33abd4bae..f0bfa0a96 100644 --- a/src/main/ThumbnailGeneration.tsx +++ b/src/main/ThumbnailGeneration.tsx @@ -212,9 +212,10 @@ const ThumbnailActions: React.FC<{ // *index: Generate from const generate = (track: Track, index: number) => { const time = generateRefs.current[index]?.getCurrentTime(); + const timeObject = time ? { time: time.toString(), flavorType: track.flavor.type } : undefined; const uri = generateRefs.current[index]?.captureVideo(); dispatch(setThumbnail({ id: track.id, uri: uri })); - dispatch(setThumbnailTime({ id: track.id, time: time?.toString() })); + dispatch(setThumbnailTime({ id: track.id, time: timeObject })); dispatch(setHasChanges(true)); }; diff --git a/src/main/ThumbnailSelect.tsx b/src/main/ThumbnailSelect.tsx index 0bb1dec9f..cd84eda44 100644 --- a/src/main/ThumbnailSelect.tsx +++ b/src/main/ThumbnailSelect.tsx @@ -351,7 +351,10 @@ export const UseForAllTracksButton: React.FC<{ const tracks = useAppSelector(selectTracks); // Set the given thumbnail for all tracks - const setForOtherThumbnails = ({ uri, time }: {uri: string | undefined, time: string | undefined}) => { + const setForOtherThumbnails = ({ uri, time }: { + uri: string | undefined, + time?: { time: string; flavorType: string } + }) => { const thumbnails = []; for (const track of tracks) { thumbnails.push({ id: track.id, uri, time }); @@ -472,13 +475,19 @@ const WorkaroundThumbnailGenerator: React.FC<{ }> = ({ track }) => { const dispatch = useAppDispatch(); + const tracks = useAppSelector(selectTracks); + const thumbnailTime = track.thumbnailTime; + const thumbnailTrack = thumbnailTime + ? tracks.find(t => t.flavor.type === thumbnailTime.flavorType) + : undefined; + const ref = useRef(null); const [ready, setReady] = useState(false); const [seeked, setSeeked] = useState(false); useEffect(() => { if (ref.current && ready && track && track.thumbnailTime && !track.thumbnailUri) { - ref.current.seekTo(parseFloat(track.thumbnailTime), "seconds"); + ref.current.seekTo(parseFloat(track.thumbnailTime.time), "seconds"); } }, [dispatch, ready, track]); @@ -505,8 +514,12 @@ const WorkaroundThumbnailGenerator: React.FC<{ display: "none", }); + if (!thumbnailTrack) { + return null; + } + return ( -