diff --git a/README.md b/README.md index 8a2c8436..1be66de2 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,13 @@ Browse and install community extensions from the [Recordly Marketplace](https:// - Feedback and issue links from the editor - Project persistence for editor preferences - Faster preview recovery after export + +### Performance and Reliability + +- Stream browser-captured recording chunks to disk to avoid large in-memory video buffers +- Keep editor playback responsive during long recordings with throttled timeline state updates +- Cache caption, zoom, and cursor lookups used by preview rendering + --- # Screenshots diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index a2f81889..61405475 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -278,6 +278,72 @@ interface Window { videoData: ArrayBuffer, fileName: string, ) => Promise<{ success: boolean; path?: string; message?: string }>; + openRecordingStream: (fileName: string) => Promise<{ + success: boolean; + streamId?: string; + path?: string; + error?: string; + }>; + writeRecordingStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; + closeRecordingStream: ( + streamId: string, + options?: { abort?: boolean; mimeType?: string }, + ) => Promise<{ + success: boolean; + path?: string; + message?: string; + error?: string; + bytesWritten?: number; + }>; + openMicrophoneSidecarStream: () => Promise<{ + success: boolean; + streamId?: string; + error?: string; + }>; + writeMicrophoneSidecarStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; + closeMicrophoneSidecarStream: ( + streamId: string, + videoPath: string, + options?: { + abort?: boolean; + startDelayMs?: number; + browserMicrophoneProfile?: string; + requestedBrowserMicrophoneProfile?: string | null; + requestedConstraints?: unknown; + mediaTrackSettings?: Record; + audioInputDevices?: Array<{ + deviceId: string; + groupId?: string; + label: string; + }>; + mediaRecorder?: { + mimeType?: string; + audioBitsPerSecond?: number; + timesliceMs?: number; + }; + chunkEvents?: Array<{ + index: number; + size: number; + elapsedMs: number; + deltaMs: number | null; + recordedElapsedMs?: number; + recordedDeltaMs?: number | null; + }>; + pauseIntervals?: Array<{ + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; + }>; + }, + ) => Promise<{ success: boolean; path?: string; error?: string }>; storeMicrophoneSidecar: ( audioData: ArrayBuffer, videoPath: string, diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index feb6aefb..f76bcf69 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -1,5 +1,6 @@ import type { ChildProcessWithoutNullStreams } from "node:child_process"; import { execFile, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; @@ -150,6 +151,156 @@ import { resolveWindowsCaptureDisplay } from "../windowsCaptureSelection"; const execFileAsync = promisify(execFile); +type BrowserMicrophoneSidecarOptions = { + startDelayMs?: number; + browserMicrophoneProfile?: string; + requestedBrowserMicrophoneProfile?: string | null; + requestedConstraints?: unknown; + mediaTrackSettings?: Record; + audioInputDevices?: unknown; + mediaRecorder?: unknown; + chunkEvents?: unknown; + pauseIntervals?: unknown; +}; + +type FileStreamSession = { + streamId: string; + fileHandle: Awaited>; + tempPath: string; + bytesWritten: number; + highestWatermark: number; + writeQueue: Promise; + aborted: boolean; +}; + +type RecordingFileStreamSession = FileStreamSession & { + finalPath: string; +}; + +type RecordingStreamCloseOptions = { + abort?: boolean; + mimeType?: string | null; +}; + +const recordingFileStreams = new Map(); +const microphoneSidecarStreams = new Map(); + +function getSafeRecordingFileName(fileName: string) { + const sanitizedBaseName = path.basename(fileName).replace(/[<>:"|?*]/g, "_"); + const baseName = Array.from(sanitizedBaseName, (char) => + char.charCodeAt(0) < 32 ? "_" : char, + ).join(""); + return baseName || `recording-${Date.now()}.webm`; +} + +async function openWritableStreamSession(tempPath: string): Promise { + await fs.mkdir(path.dirname(tempPath), { recursive: true }); + const streamId = `recordly-recording-stream-${randomUUID()}`; + const fileHandle = await fs.open(tempPath, "w"); + return { + streamId, + fileHandle, + tempPath, + bytesWritten: 0, + highestWatermark: 0, + writeQueue: Promise.resolve(), + aborted: false, + }; +} + +async function writeStreamSessionChunk( + sessions: Map, + streamId: string, + position: number, + chunk: Uint8Array, +) { + const session = sessions.get(streamId); + if (!session) { + throw new Error(`Recording stream not found: ${streamId}`); + } + if (session.aborted) { + throw new Error("Recording stream was aborted"); + } + + const next = session.writeQueue.then(async () => { + if (session.aborted) return; + const buffer = Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength); + await session.fileHandle.write(buffer, 0, buffer.byteLength, position); + session.bytesWritten += buffer.byteLength; + session.highestWatermark = Math.max(session.highestWatermark, position + buffer.byteLength); + }); + session.writeQueue = next.catch(() => undefined); + await next; +} + +async function closeStreamSession( + sessions: Map, + streamId: string, + options?: { abort?: boolean }, +) { + const session = sessions.get(streamId); + if (!session) { + throw new Error(`Recording stream not found: ${streamId}`); + } + + if (options?.abort) { + session.aborted = true; + } + + try { + await session.writeQueue; + } finally { + await session.fileHandle.close().catch(() => undefined); + sessions.delete(streamId); + } + + if (session.aborted) { + await fs.rm(session.tempPath, { force: true }).catch(() => undefined); + } + + return session; +} + +function isWebmRecording(filePath: string, mimeType?: string | null) { + return /^video\/webm(?:[;\s]|$)/i.test(mimeType ?? "") || /\.webm$/i.test(filePath); +} + +async function remuxStreamedWebmRecording( + videoPath: string, + options?: RecordingStreamCloseOptions, +) { + if (!isWebmRecording(videoPath, options?.mimeType)) { + return; + } + + const remuxedPath = `${videoPath}.${randomUUID()}.fixed.webm`; + try { + await execFileAsync( + getFfmpegBinaryPath(), + [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + videoPath, + "-map", + "0", + "-c", + "copy", + "-avoid_negative_ts", + "make_zero", + remuxedPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ); + await moveFileWithOverwrite(remuxedPath, videoPath); + } catch (error) { + await fs.rm(remuxedPath, { force: true }).catch(() => undefined); + console.warn("Failed to remux streamed WebM recording:", error); + } +} + async function writeWindowsRecordingDiagnostics( videoPath: string | null | undefined, snapshot: Omit, @@ -366,6 +517,136 @@ async function cleanupWindowsOrphanedMicAudioPath(filePath: string | null) { await fs.rm(filePath, { force: true }).catch(() => undefined); } +async function finalizeMicrophoneSidecarFromWebm( + tempWebmPath: string, + videoPath: string, + options?: BrowserMicrophoneSidecarOptions, +) { + const baseName = videoPath.replace(/\.[^.]+$/, ""); + const sidecarPath = `${baseName}.mic.wav`; + const sourceWebmPath = `${baseName}.mic.source.webm`; + const sourceBytes = await getFileSizeIfPresent(tempWebmPath); + + try { + await execFileAsync( + getFfmpegBinaryPath(), + [ + "-y", + "-hide_banner", + "-nostdin", + "-nostats", + "-i", + tempWebmPath, + "-vn", + "-ac", + "1", + "-ar", + "48000", + "-af", + [ + ...getBrowserMicSidecarFilters(options?.browserMicrophoneProfile), + "aresample=async=1:first_pts=0", + ].join(","), + "-c:a", + "pcm_s16le", + sidecarPath, + ], + { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, + ); + if (shouldKeepRecordingAudioSidecars()) { + await fs.rename(tempWebmPath, sourceWebmPath).catch(async () => { + await fs.copyFile(tempWebmPath, sourceWebmPath); + await fs.rm(tempWebmPath, { force: true }); + }); + } else { + await fs.rm(tempWebmPath, { force: true }); + } + const startDelayMs = options?.startDelayMs; + const mediaTrackSettings = pickPrimitiveRecord(options?.mediaTrackSettings); + const audioInputDevices = pickAudioInputDevices(options?.audioInputDevices); + const mediaRecorder = isRecord(options?.mediaRecorder) + ? { + ...(typeof options.mediaRecorder.mimeType === "string" + ? { mimeType: options.mediaRecorder.mimeType } + : {}), + ...(typeof options.mediaRecorder.audioBitsPerSecond === "number" + ? { + audioBitsPerSecond: Math.round( + options.mediaRecorder.audioBitsPerSecond, + ), + } + : {}), + ...(typeof options.mediaRecorder.timesliceMs === "number" + ? { timesliceMs: Math.round(options.mediaRecorder.timesliceMs) } + : {}), + } + : null; + const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents); + const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals); + const chunkTiming = + chunkEvents || pauseIntervals + ? summarizeMicrophoneChunkTiming( + chunkEvents, + pauseIntervals, + mediaRecorder?.timesliceMs, + ) + : null; + const metadata = { + ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 + ? { startDelayMs: Math.round(startDelayMs ?? 0) } + : {}), + ...(typeof options?.browserMicrophoneProfile === "string" + ? { browserMicrophoneProfile: options.browserMicrophoneProfile } + : {}), + ...(typeof options?.requestedBrowserMicrophoneProfile === "string" + ? { + requestedBrowserMicrophoneProfile: + options.requestedBrowserMicrophoneProfile, + } + : {}), + ...(isRecord(options?.requestedConstraints) + ? { requestedConstraints: options.requestedConstraints } + : {}), + ...(mediaTrackSettings ? { mediaTrackSettings } : {}), + ...(audioInputDevices ? { audioInputDevices } : {}), + ...(mediaRecorder && Object.keys(mediaRecorder).length > 0 + ? { mediaRecorder } + : {}), + ...(chunkEvents ? { chunkEvents } : {}), + ...(pauseIntervals ? { pauseIntervals } : {}), + ...(chunkTiming ? { chunkTiming } : {}), + }; + if (Object.keys(metadata).length > 0) { + try { + await fs.writeFile(`${sidecarPath}.json`, JSON.stringify(metadata)); + } catch (metadataError) { + console.warn("Failed to store microphone sidecar timing metadata:", metadataError); + } + } + await writeRecordingDiagnosticsSnapshot(videoPath, { + backend: "browser-store", + phase: "mic-sidecar", + outputPath: videoPath, + microphonePath: sidecarPath, + details: { + sourceBytes, + sourceWebmPath: shouldKeepRecordingAudioSidecars() ? sourceWebmPath : null, + metadata, + }, + }).catch((diagnosticsError) => { + console.warn("Failed to write microphone sidecar diagnostics:", diagnosticsError); + }); + return { success: true, path: sidecarPath }; + } catch (error) { + await Promise.all([ + fs.rm(tempWebmPath, { force: true }).catch(() => undefined), + fs.rm(sidecarPath, { force: true }).catch(() => undefined), + ]); + console.error("Failed to store microphone sidecar:", error); + return { success: false, error: String(error) }; + } +} + export function registerRecordingHandlers( onRecordingStateChange?: (recording: boolean, sourceName: string) => void, ) { @@ -1468,154 +1749,115 @@ export function registerRecordingHandlers( } }); + ipcMain.handle("recording-stream-open", async (_event, fileName: string) => { + try { + const recordingsDir = await getRecordingsDir(); + const safeFileName = getSafeRecordingFileName(fileName); + const finalPath = path.join(recordingsDir, safeFileName); + const tempPath = path.join(recordingsDir, `${safeFileName}.${randomUUID()}.tmp`); + const session = (await openWritableStreamSession(tempPath)) as RecordingFileStreamSession; + session.finalPath = finalPath; + recordingFileStreams.set(session.streamId, session); + return { success: true, streamId: session.streamId, path: finalPath }; + } catch (error) { + return { success: false, error: String(error) }; + } + }); + + ipcMain.handle( + "recording-stream-write", + async (_event, streamId: string, position: number, chunk: Uint8Array) => { + try { + await writeStreamSessionChunk(recordingFileStreams, streamId, position, chunk); + return { success: true }; + } catch (error) { + return { success: false, error: String(error) }; + } + }, + ); + + ipcMain.handle( + "recording-stream-close", + async (_event, streamId: string, options?: RecordingStreamCloseOptions) => { + try { + const session = await closeStreamSession(recordingFileStreams, streamId, options); + if (options?.abort) { + return { success: true, bytesWritten: 0 }; + } + await moveFileWithOverwrite(session.tempPath, session.finalPath); + await remuxStreamedWebmRecording(session.finalPath, options); + const result = await finalizeStoredVideo(session.finalPath); + return { ...result, bytesWritten: session.highestWatermark }; + } catch (error) { + return { success: false, error: String(error), message: "Failed to store video" }; + } + }, + ); + + ipcMain.handle("microphone-sidecar-stream-open", async () => { + try { + const recordingsDir = await getRecordingsDir(); + const tempPath = path.join( + recordingsDir, + `recordly-microphone-${Date.now()}-${randomUUID()}.source.webm.tmp`, + ); + const session = await openWritableStreamSession(tempPath); + microphoneSidecarStreams.set(session.streamId, session); + return { success: true, streamId: session.streamId }; + } catch (error) { + return { success: false, error: String(error) }; + } + }); + + ipcMain.handle( + "microphone-sidecar-stream-write", + async (_event, streamId: string, position: number, chunk: Uint8Array) => { + try { + await writeStreamSessionChunk(microphoneSidecarStreams, streamId, position, chunk); + return { success: true }; + } catch (error) { + return { success: false, error: String(error) }; + } + }, + ); + + ipcMain.handle( + "microphone-sidecar-stream-close", + async ( + _event, + streamId: string, + videoPath: string, + options?: BrowserMicrophoneSidecarOptions & { abort?: boolean }, + ) => { + try { + const session = await closeStreamSession(microphoneSidecarStreams, streamId, options); + if (options?.abort) { + return { success: true }; + } + return await finalizeMicrophoneSidecarFromWebm(session.tempPath, videoPath, options); + } catch (error) { + return { success: false, error: String(error) }; + } + }, + ); + ipcMain.handle( "store-microphone-sidecar", async ( _, audioData: ArrayBuffer, videoPath: string, - options?: { - startDelayMs?: number; - browserMicrophoneProfile?: string; - requestedBrowserMicrophoneProfile?: string | null; - requestedConstraints?: unknown; - mediaTrackSettings?: Record; - audioInputDevices?: unknown; - mediaRecorder?: unknown; - chunkEvents?: unknown; - pauseIntervals?: unknown; - }, + options?: BrowserMicrophoneSidecarOptions, ) => { const baseName = videoPath.replace(/\.[^.]+$/, ""); - const sidecarPath = `${baseName}.mic.wav`; - const sourceWebmPath = `${baseName}.mic.source.webm`; - const tempWebmPath = `${sourceWebmPath}.tmp`; - + const tempWebmPath = `${baseName}.mic.source.webm.tmp`; try { await fs.writeFile(tempWebmPath, Buffer.from(audioData)); - await execFileAsync( - getFfmpegBinaryPath(), - [ - "-y", - "-hide_banner", - "-nostdin", - "-nostats", - "-i", - tempWebmPath, - "-vn", - "-ac", - "1", - "-ar", - "48000", - "-af", - [ - ...getBrowserMicSidecarFilters(options?.browserMicrophoneProfile), - "aresample=async=1:first_pts=0", - ].join(","), - "-c:a", - "pcm_s16le", - sidecarPath, - ], - { timeout: 120000, maxBuffer: 10 * 1024 * 1024 }, - ); - if (shouldKeepRecordingAudioSidecars()) { - await fs.rename(tempWebmPath, sourceWebmPath).catch(async () => { - await fs.copyFile(tempWebmPath, sourceWebmPath); - await fs.rm(tempWebmPath, { force: true }); - }); - } else { - await fs.rm(tempWebmPath, { force: true }); - } - const startDelayMs = options?.startDelayMs; - const mediaTrackSettings = pickPrimitiveRecord(options?.mediaTrackSettings); - const audioInputDevices = pickAudioInputDevices(options?.audioInputDevices); - const mediaRecorder = isRecord(options?.mediaRecorder) - ? { - ...(typeof options.mediaRecorder.mimeType === "string" - ? { mimeType: options.mediaRecorder.mimeType } - : {}), - ...(typeof options.mediaRecorder.audioBitsPerSecond === "number" - ? { - audioBitsPerSecond: Math.round( - options.mediaRecorder.audioBitsPerSecond, - ), - } - : {}), - ...(typeof options.mediaRecorder.timesliceMs === "number" - ? { timesliceMs: Math.round(options.mediaRecorder.timesliceMs) } - : {}), - } - : null; - const chunkEvents = pickMicrophoneChunkEvents(options?.chunkEvents); - const pauseIntervals = pickMicrophonePauseIntervals(options?.pauseIntervals); - const chunkTiming = - chunkEvents || pauseIntervals - ? summarizeMicrophoneChunkTiming( - chunkEvents, - pauseIntervals, - mediaRecorder?.timesliceMs, - ) - : null; - const metadata = { - ...(Number.isFinite(startDelayMs) && (startDelayMs ?? 0) >= 0 - ? { startDelayMs: Math.round(startDelayMs ?? 0) } - : {}), - ...(typeof options?.browserMicrophoneProfile === "string" - ? { browserMicrophoneProfile: options.browserMicrophoneProfile } - : {}), - ...(typeof options?.requestedBrowserMicrophoneProfile === "string" - ? { - requestedBrowserMicrophoneProfile: - options.requestedBrowserMicrophoneProfile, - } - : {}), - ...(isRecord(options?.requestedConstraints) - ? { requestedConstraints: options.requestedConstraints } - : {}), - ...(mediaTrackSettings ? { mediaTrackSettings } : {}), - ...(audioInputDevices ? { audioInputDevices } : {}), - ...(mediaRecorder && Object.keys(mediaRecorder).length > 0 - ? { mediaRecorder } - : {}), - ...(chunkEvents ? { chunkEvents } : {}), - ...(pauseIntervals ? { pauseIntervals } : {}), - ...(chunkTiming ? { chunkTiming } : {}), - }; - if (Object.keys(metadata).length > 0) { - try { - await fs.writeFile(`${sidecarPath}.json`, JSON.stringify(metadata)); - } catch (metadataError) { - console.warn( - "Failed to store microphone sidecar timing metadata:", - metadataError, - ); - } - } - await writeRecordingDiagnosticsSnapshot(videoPath, { - backend: "browser-store", - phase: "mic-sidecar", - outputPath: videoPath, - microphonePath: sidecarPath, - details: { - sourceBytes: audioData.byteLength, - sourceWebmPath: shouldKeepRecordingAudioSidecars() ? sourceWebmPath : null, - metadata, - }, - }).catch((diagnosticsError) => { - console.warn( - "Failed to write microphone sidecar diagnostics:", - diagnosticsError, - ); - }); - return { success: true, path: sidecarPath }; } catch (error) { - await Promise.all([ - fs.rm(tempWebmPath, { force: true }).catch(() => undefined), - fs.rm(sidecarPath, { force: true }).catch(() => undefined), - ]); console.error("Failed to store microphone sidecar:", error); return { success: false, error: String(error) }; } + return await finalizeMicrophoneSidecarFromWebm(tempWebmPath, videoPath, options); }, ); diff --git a/electron/preload.ts b/electron/preload.ts index c823de71..5cb2ef8e 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -509,6 +509,70 @@ contextBridge.exposeInMainWorld("electronAPI", { storeRecordedVideo: (videoData: ArrayBuffer, fileName: string) => { return ipcRenderer.invoke("store-recorded-video", videoData, fileName); }, + openRecordingStream: (fileName: string) => { + return ipcRenderer.invoke("recording-stream-open", fileName); + }, + writeRecordingStreamChunk: (streamId: string, position: number, chunk: Uint8Array) => { + return ipcRenderer.invoke("recording-stream-write", streamId, position, chunk); + }, + closeRecordingStream: ( + streamId: string, + options?: { abort?: boolean; mimeType?: string }, + ) => { + return ipcRenderer.invoke("recording-stream-close", streamId, options); + }, + openMicrophoneSidecarStream: () => { + return ipcRenderer.invoke("microphone-sidecar-stream-open"); + }, + writeMicrophoneSidecarStreamChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => { + return ipcRenderer.invoke("microphone-sidecar-stream-write", streamId, position, chunk); + }, + closeMicrophoneSidecarStream: ( + streamId: string, + videoPath: string, + options?: { + abort?: boolean; + startDelayMs?: number; + browserMicrophoneProfile?: string; + requestedBrowserMicrophoneProfile?: string | null; + requestedConstraints?: unknown; + mediaTrackSettings?: Record; + audioInputDevices?: Array<{ + deviceId: string; + groupId?: string; + label: string; + }>; + mediaRecorder?: { + mimeType?: string; + audioBitsPerSecond?: number; + timesliceMs?: number; + }; + chunkEvents?: Array<{ + index: number; + size: number; + elapsedMs: number; + deltaMs: number | null; + recordedElapsedMs?: number; + recordedDeltaMs?: number | null; + }>; + pauseIntervals?: Array<{ + startElapsedMs: number; + endElapsedMs?: number; + durationMs?: number; + }>; + }, + ) => { + return ipcRenderer.invoke( + "microphone-sidecar-stream-close", + streamId, + videoPath, + options, + ); + }, storeMicrophoneSidecar: ( audioData: ArrayBuffer, videoPath: string, diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index c64e6b42..55895d70 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -206,6 +206,7 @@ type PixiRendererAttempt = { message: string; }; const PIXI_RENDERER_INIT_TIMEOUT_MS = 8_000; +const REACT_PLAYBACK_TIME_UPDATE_INTERVAL_MS = 80; function isCanvasRenderer(application: Application): boolean { const rendererName = application?.renderer?.constructor?.name?.toLowerCase(); @@ -263,21 +264,46 @@ function getCursorPositionAtTime( return null; } - let closest = telemetry[0]; - let minDist = Math.abs(telemetry[0].timeMs - timeMs); + if (timeMs <= telemetry[0].timeMs) { + const first = telemetry[0]; + return mapCursorToCanvasNormalized( + { + cx: first.cx, + cy: first.cy, + interactionType: first.interactionType, + }, + params ?? { canvasWidth: 1, canvasHeight: 1 }, + ); + } - for (let index = 1; index < telemetry.length; index++) { - const point = telemetry[index]; - const distance = Math.abs(point.timeMs - timeMs); - if (distance < minDist) { - minDist = distance; - closest = point; - } - if (point.timeMs > timeMs) { - break; + if (timeMs >= telemetry[telemetry.length - 1].timeMs) { + const last = telemetry[telemetry.length - 1]; + return mapCursorToCanvasNormalized( + { + cx: last.cx, + cy: last.cy, + interactionType: last.interactionType, + }, + params ?? { canvasWidth: 1, canvasHeight: 1 }, + ); + } + + let lo = 0; + let hi = telemetry.length - 1; + while (lo < hi - 1) { + const mid = (lo + hi) >> 1; + if (telemetry[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid; } } + const before = telemetry[lo]; + const after = telemetry[hi]; + const closest = + Math.abs(before.timeMs - timeMs) <= Math.abs(after.timeMs - timeMs) ? before : after; + return mapCursorToCanvasNormalized( { cx: closest.cx, @@ -1314,12 +1340,24 @@ const VideoPlayback = forwardRef( const bgVideo = bgVideoRef.current; if (bgVideo) { if (isPlaying) { - bgVideo.play().catch(() => undefined); + if (bgVideo.paused) { + bgVideo.play().catch(() => undefined); + } } else { bgVideo.pause(); } } - }, [isPlaying]); + const webcamVideo = webcamVideoRef.current; + if (webcamVideo) { + if (isPlaying && webcamEnabled && webcamVideoPath) { + if (webcamVideo.paused) { + webcamVideo.play().catch(() => undefined); + } + } else { + webcamVideo.pause(); + } + } + }, [isPlaying, webcamEnabled, webcamVideoPath]); useEffect(() => { suspendRenderingRef.current = suspendRendering; @@ -1413,15 +1451,6 @@ const VideoPlayback = forwardRef( } } - if (isPlaying) { - const playPromise = bgVideo.play(); - if (playPromise) { - playPromise.catch(() => undefined); - } - } else { - bgVideo.pause(); - } - lastBackgroundSyncTimeRef.current = clipTimelineTime; }, [currentTime, isPlaying]); @@ -1738,15 +1767,6 @@ const VideoPlayback = forwardRef( } } - if (isPlaying) { - const playPromise = webcamVideo.play(); - if (playPromise) { - playPromise.catch(() => undefined); - } - } else { - webcamVideo.pause(); - } - lastWebcamSyncTimeRef.current = targetTime; }, [currentTime, isPlaying, webcamEnabled, webcamTimeOffsetMs, webcamVideoPath]); @@ -1996,6 +2016,7 @@ const VideoPlayback = forwardRef( timeUpdateAnimationRef, onPlayStateChange, onTimeUpdate, + onTimeUpdateMinIntervalMs: REACT_PLAYBACK_TIME_UPDATE_INTERVAL_MS, trimRegionsRef, speedRegionsRef, }); diff --git a/src/components/video-editor/captionLayout.ts b/src/components/video-editor/captionLayout.ts index 6c0d987e..9a08eca5 100644 --- a/src/components/video-editor/captionLayout.ts +++ b/src/components/video-editor/captionLayout.ts @@ -48,6 +48,13 @@ export interface ActiveCaptionLayout { scale: number; } +interface CaptionStaticLayout { + sourceWords: ReturnType; + lines: CaptionLineLayout[]; + pages: CaptionPageLayout[]; + maxRows: number; +} + type CaptionSourceWord = { cueId: string; cueWordIndex: number; @@ -61,6 +68,7 @@ type CaptionSourceWord = { const CAPTION_ENTER_MS = 180; const CAPTION_EXIT_MS = 140; const CAPTION_BLOCK_GAP_BREAK_MS = 500; +const captionStaticLayoutCache = new WeakMap>(); function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); @@ -405,31 +413,83 @@ function getVisibleCaptionText(lines: CaptionLineLayout[]) { .join(" "); } -export function buildActiveCaptionLayout(options: { +function getCaptionLayoutCacheKey(settings: AutoCaptionSettings, maxWidthPx: number) { + return [ + Math.round(maxWidthPx * 100) / 100, + clamp(Math.round(settings.maxRows || 1), 1, 4), + ].join(":"); +} + +function getCaptionWordState(index: number, activeWordIndex: number): CaptionWordState { + return index < activeWordIndex + ? "spoken" + : index === activeWordIndex + ? "active" + : "upcoming"; +} + +function applyCaptionWordStates( + lines: CaptionLineLayout[], + activeWordIndex: number, +): CaptionLineLayout[] { + return lines.map((line) => ({ + ...line, + words: line.words.map((word) => ({ + ...word, + state: getCaptionWordState(word.index, activeWordIndex), + })), + })); +} + +function findActiveCaptionWordIndex( + sourceWords: CaptionStaticLayout["sourceWords"], + timeMs: number, +) { + if (sourceWords.length === 0) { + return -1; + } + + let lo = 0; + let hi = sourceWords.length - 1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + const word = sourceWords[mid]; + if (timeMs < word.startMs) { + hi = mid - 1; + } else if (timeMs >= word.endMs) { + lo = mid + 1; + } else { + return mid; + } + } + + return clamp(hi, 0, sourceWords.length - 1); +} + +function getOrBuildCaptionStaticLayout(options: { cues: CaptionCue[]; - timeMs: number; settings: AutoCaptionSettings; maxWidthPx: number; measureText: (text: string) => number; -}) { +}): CaptionStaticLayout | null { + let cueCache = captionStaticLayoutCache.get(options.cues); + if (!cueCache) { + cueCache = new Map(); + captionStaticLayoutCache.set(options.cues, cueCache); + } + + const cacheKey = getCaptionLayoutCacheKey(options.settings, options.maxWidthPx); + const cached = cueCache.get(cacheKey); + if (cached) { + return cached; + } + const sourceWords = flattenCaptionWords(options.cues); if (sourceWords.length === 0) { return null; } - let activeWordIndex = -1; - activeWordIndex = sourceWords.findIndex( - (word) => options.timeMs >= word.startMs && options.timeMs < word.endMs, - ); - if (activeWordIndex < 0) { - activeWordIndex = sourceWords.findIndex((word) => options.timeMs < word.startMs); - activeWordIndex = - activeWordIndex < 0 - ? sourceWords.length - 1 - : clamp(activeWordIndex - 1, 0, sourceWords.length - 1); - } const maxRows = clamp(Math.round(options.settings.maxRows || 1), 1, 4); - const words: CaptionWordLayout[] = sourceWords.map((word, index) => { return { cueId: word.cueId, @@ -441,12 +501,7 @@ export function buildActiveCaptionLayout(options: { startMs: word.startMs, endMs: word.endMs, hasRealTiming: word.hasRealTiming, - state: - index < activeWordIndex - ? "spoken" - : index === activeWordIndex - ? "active" - : "upcoming", + state: "upcoming", }; }); @@ -468,13 +523,39 @@ export function buildActiveCaptionLayout(options: { text: "", }, }); + + const layout = { sourceWords, lines, pages, maxRows }; + cueCache.set(cacheKey, layout); + return layout; +} + +export function buildActiveCaptionLayout(options: { + cues: CaptionCue[]; + timeMs: number; + settings: AutoCaptionSettings; + maxWidthPx: number; + measureText: (text: string) => number; +}) { + const staticLayout = getOrBuildCaptionStaticLayout(options); + if (!staticLayout) { + return null; + } + + const { sourceWords, lines, pages, maxRows } = staticLayout; + const activeWordIndex = findActiveCaptionWordIndex(sourceWords, options.timeMs); const visiblePageIndex = getVisibleCaptionPageIndex(pages, options.timeMs); if (visiblePageIndex < 0) { return null; } const visiblePage = pages[visiblePageIndex] ?? null; - const visibleLines = visiblePage?.lines ?? lines.slice(0, maxRows); - const activeWord = activeWordIndex >= 0 ? words[activeWordIndex] : null; + const visibleLines = applyCaptionWordStates( + visiblePage?.lines ?? lines.slice(0, maxRows), + activeWordIndex, + ); + const activeWord = + activeWordIndex >= 0 + ? visibleLines.flatMap((line) => line.words).find((word) => word.index === activeWordIndex) + : null; const activeWordProgress = activeWord ? clamp01( (options.timeMs - activeWord.startMs) / diff --git a/src/components/video-editor/videoPlayback/cursorRenderer.ts b/src/components/video-editor/videoPlayback/cursorRenderer.ts index 644cf899..a7926e14 100644 --- a/src/components/video-editor/videoPlayback/cursorRenderer.ts +++ b/src/components/video-editor/videoPlayback/cursorRenderer.ts @@ -682,11 +682,28 @@ function findLatestSample(samples: CursorTelemetryPoint[], timeMs: number) { } function findLatestInteractionSample(samples: CursorTelemetryPoint[], timeMs: number) { - for (let index = samples.length - 1; index >= 0; index -= 1) { + if (samples.length === 0) return null; + + const oldestRelevantTimeMs = timeMs - CLICK_RING_FADE_MS; + let lo = 0; + let hi = samples.length - 1; + while (lo < hi) { + const mid = Math.ceil((lo + hi) / 2); + if (samples[mid].timeMs <= timeMs) { + lo = mid; + } else { + hi = mid - 1; + } + } + + for (let index = lo; index >= 0; index -= 1) { const sample = samples[index]; if (sample.timeMs > timeMs) { continue; } + if (sample.timeMs < oldestRelevantTimeMs) { + break; + } if ( sample.interactionType === "click" || diff --git a/src/components/video-editor/videoPlayback/videoEventHandlers.ts b/src/components/video-editor/videoPlayback/videoEventHandlers.ts index b85bf00d..b46a8fae 100644 --- a/src/components/video-editor/videoPlayback/videoEventHandlers.ts +++ b/src/components/video-editor/videoPlayback/videoEventHandlers.ts @@ -23,6 +23,7 @@ interface VideoEventHandlersParams { timeUpdateAnimationRef: React.MutableRefObject; onPlayStateChange: (playing: boolean) => void; onTimeUpdate: (time: number) => void; + onTimeUpdateMinIntervalMs?: number; trimRegionsRef: React.MutableRefObject; speedRegionsRef: React.MutableRefObject; } @@ -37,17 +38,34 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { timeUpdateAnimationRef, onPlayStateChange, onTimeUpdate, + onTimeUpdateMinIntervalMs = 0, trimRegionsRef, speedRegionsRef, } = params; const presentedFrameVideo = video as PresentedFrameVideoElement; let videoFrameRequestId: number | null = null; + let lastReactTimeUpdateAtMs = 0; + let lastReactTimeUpdateValue = Number.NaN; enablePitchPreservingPlayback(video); - const emitTime = (timeValue: number) => { + const emitTime = (timeValue: number, options?: { forceReactUpdate?: boolean }) => { currentTimeRef.current = timeValue * 1000; - onTimeUpdate(timeValue); extensionHost.emitEvent({ type: "playback:timeupdate", timeMs: timeValue * 1000 }); + + const now = performance.now(); + const timeJumped = + Number.isFinite(lastReactTimeUpdateValue) && + Math.abs(timeValue - lastReactTimeUpdateValue) > 0.25; + if ( + options?.forceReactUpdate || + onTimeUpdateMinIntervalMs <= 0 || + now - lastReactTimeUpdateAtMs >= onTimeUpdateMinIntervalMs || + timeJumped + ) { + lastReactTimeUpdateAtMs = now; + lastReactTimeUpdateValue = timeValue; + onTimeUpdate(timeValue); + } }; // Helper function to check if current time is within a trim region @@ -74,7 +92,7 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { const clampedSkipToTime = Math.min(skipToTime, video.duration); video.currentTime = clampedSkipToTime; - emitTime(clampedSkipToTime); + emitTime(clampedSkipToTime, { forceReactUpdate: true }); if (clampedSkipToTime >= video.duration) { video.pause(); @@ -161,7 +179,7 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { isPlayingRef.current = false; onPlayStateChange(false); cancelScheduledUpdate(); - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); }; const handleSeeked = () => { @@ -174,13 +192,13 @@ export function createVideoEventHandlers(params: VideoEventHandlersParams) { if (activeTrimRegion) { skipPastTrimRegion(activeTrimRegion); } else { - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); } }; const handleSeeking = () => { isSeekingRef.current = true; - emitTime(video.currentTime); + emitTime(video.currentTime, { forceReactUpdate: true }); }; return { diff --git a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts index 4566455d..4bdef569 100644 --- a/src/components/video-editor/videoPlayback/zoomRegionUtils.ts +++ b/src/components/video-editor/videoPlayback/zoomRegionUtils.ts @@ -34,6 +34,14 @@ type ConnectedPanTransition = { endScale: number; }; +type ConnectedRegionCache = { + pairs: ConnectedRegionPair[]; + outgoingByRegionId: Map; + incomingByRegionId: Map; +}; + +const connectedRegionCache = new WeakMap(); + function lerp(start: number, end: number, amount: number) { return start + (end - start) * amount; } @@ -115,50 +123,74 @@ function getConnectedRegionPairs(regions: ZoomRegion[]) { return pairs; } +function getConnectedRegionCache(regions: ZoomRegion[]): ConnectedRegionCache { + const cached = connectedRegionCache.get(regions); + if (cached) { + return cached; + } + + const pairs = getConnectedRegionPairs(regions); + const outgoingByRegionId = new Map(); + const incomingByRegionId = new Map(); + + for (const pair of pairs) { + outgoingByRegionId.set(pair.currentRegion.id, pair); + incomingByRegionId.set(pair.nextRegion.id, pair); + } + + const cache = { pairs, outgoingByRegionId, incomingByRegionId }; + connectedRegionCache.set(regions, cache); + return cache; +} + function getActiveRegion( regions: ZoomRegion[], timeMs: number, - connectedPairs: ConnectedRegionPair[], + connectedCache: ConnectedRegionCache, options: DominantRegionOptions, ) { - const activeRegions = regions - .map((region) => { - const outgoingPair = connectedPairs.find((pair) => pair.currentRegion.id === region.id); - if (outgoingPair && timeMs >= outgoingPair.transitionStart) { - return { region, strength: 0 }; - } - - const incomingPair = connectedPairs.find((pair) => pair.nextRegion.id === region.id); + let activeRegion: ZoomRegion | null = null; + let activeStrength = 0; + + for (const region of regions) { + let strength = 0; + const outgoingPair = connectedCache.outgoingByRegionId.get(region.id); + if (outgoingPair && timeMs >= outgoingPair.transitionStart) { + strength = 0; + } else { + const incomingPair = connectedCache.incomingByRegionId.get(region.id); if (incomingPair) { if (timeMs < incomingPair.transitionStart) { - return { region, strength: 0 }; + strength = 0; + } else { + const nextRegionZoomOutStart = + incomingPair.nextRegion.endMs - + ZOOM_OUT_EARLY_START_MS + + ZOOM_ANIMATION_LEAD_MS; + strength = timeMs < nextRegionZoomOutStart + ? 1 + : computeRegionStrength(region, timeMs, options); } - - const nextRegionZoomOutStart = - incomingPair.nextRegion.endMs - - ZOOM_OUT_EARLY_START_MS + - ZOOM_ANIMATION_LEAD_MS; - if (timeMs < nextRegionZoomOutStart) { - return { region, strength: 1 }; - } - } - - return { region, strength: computeRegionStrength(region, timeMs, options) }; - }) - .filter((entry) => entry.strength > 0) - .sort((left, right) => { - if (right.strength !== left.strength) { - return right.strength - left.strength; + } else { + strength = computeRegionStrength(region, timeMs, options); } + } - return right.region.startMs - left.region.startMs; - }); + if ( + strength > 0 && + (!activeRegion || + strength > activeStrength || + (strength === activeStrength && region.startMs > activeRegion.startMs)) + ) { + activeRegion = region; + activeStrength = strength; + } + } - if (activeRegions.length === 0) { + if (!activeRegion) { return null; } - const activeRegion = activeRegions[0].region; const activeScale = ZOOM_DEPTH_SCALES[activeRegion.depth]; return { @@ -166,7 +198,7 @@ function getActiveRegion( ...activeRegion, focus: getResolvedFocus(activeRegion, activeScale), }, - strength: activeRegions[0].strength, + strength: activeStrength, blendedScale: null, }; } @@ -237,21 +269,23 @@ export function findDominantRegion( blendedScale: number | null; transition: ConnectedPanTransition | null; } { - const connectedPairs = options.connectZooms ? getConnectedRegionPairs(regions) : []; + const connectedCache = options.connectZooms + ? getConnectedRegionCache(regions) + : { pairs: [], outgoingByRegionId: new Map(), incomingByRegionId: new Map() }; if (options.connectZooms) { - const connectedTransition = getConnectedRegionTransition(connectedPairs, timeMs); + const connectedTransition = getConnectedRegionTransition(connectedCache.pairs, timeMs); if (connectedTransition) { return connectedTransition; } - const connectedHold = getConnectedRegionHold(timeMs, connectedPairs); + const connectedHold = getConnectedRegionHold(timeMs, connectedCache.pairs); if (connectedHold) { return { ...connectedHold, transition: null }; } } - const activeRegion = getActiveRegion(regions, timeMs, connectedPairs, options); + const activeRegion = getActiveRegion(regions, timeMs, connectedCache, options); return activeRegion ? { ...activeRegion, transition: null } : { region: null, strength: 0, blendedScale: null, transition: null }; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 6f021761..f3a99bb2 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -107,6 +107,150 @@ type MicrophoneSidecarOptions = { chunkEvents?: MicrophoneFallbackChunkEvent[]; pauseIntervals?: MicrophoneFallbackPauseInterval[]; }; + +type ChunkStreamWriter = { + readonly path?: string; + write: (blob: Blob) => void; + close: (options?: { + mimeType?: string; + }) => Promise<{ success: boolean; path?: string; message?: string; error?: string }>; + abort: () => Promise; +}; + +type MicrophoneSidecarStreamWriter = { + write: (blob: Blob) => void; + close: ( + videoPath: string, + options?: MicrophoneSidecarOptions, + ) => Promise<{ success: boolean; path?: string; error?: string }>; + abort: () => Promise; +}; + +function createQueuedBlobWriter(params: { + streamId: string; + writeChunk: ( + streamId: string, + position: number, + chunk: Uint8Array, + ) => Promise<{ success: boolean; error?: string }>; +}) { + let position = 0; + let writeQueue = Promise.resolve(); + let writeError: Error | null = null; + + const write = (blob: Blob) => { + if (blob.size <= 0 || writeError) { + return; + } + + writeQueue = writeQueue + .then(async () => { + const buffer = await blob.arrayBuffer(); + const bytes = new Uint8Array(buffer); + const writePosition = position; + position += bytes.byteLength; + const result = await params.writeChunk(params.streamId, writePosition, bytes); + if (!result.success) { + throw new Error(result.error || "Failed to write recording chunk"); + } + }) + .catch((error) => { + writeError = error instanceof Error ? error : new Error(String(error)); + }); + }; + + const waitForWrites = async () => { + await writeQueue; + if (writeError) { + throw writeError; + } + }; + + return { write, waitForWrites }; +} + +async function createRecordingStreamWriter(fileName: string): Promise { + if ( + !window.electronAPI?.openRecordingStream || + !window.electronAPI?.writeRecordingStreamChunk || + !window.electronAPI?.closeRecordingStream + ) { + return null; + } + + const openResult = await window.electronAPI.openRecordingStream(fileName); + if (!openResult.success || !openResult.streamId) { + console.warn("Recording stream unavailable:", openResult.error); + return null; + } + + const writer = createQueuedBlobWriter({ + streamId: openResult.streamId, + writeChunk: window.electronAPI.writeRecordingStreamChunk, + }); + + return { + path: openResult.path, + write: writer.write, + close: async (options) => { + try { + await writer.waitForWrites(); + return window.electronAPI.closeRecordingStream(openResult.streamId!, options); + } catch (error) { + await window.electronAPI.closeRecordingStream(openResult.streamId!, { abort: true }); + throw error; + } + }, + abort: async () => { + await window.electronAPI.closeRecordingStream(openResult.streamId!, { abort: true }); + }, + }; +} + +async function createMicrophoneSidecarStreamWriter(): Promise { + if ( + !window.electronAPI?.openMicrophoneSidecarStream || + !window.electronAPI?.writeMicrophoneSidecarStreamChunk || + !window.electronAPI?.closeMicrophoneSidecarStream + ) { + return null; + } + + const openResult = await window.electronAPI.openMicrophoneSidecarStream(); + if (!openResult.success || !openResult.streamId) { + console.warn("Microphone sidecar stream unavailable:", openResult.error); + return null; + } + + const writer = createQueuedBlobWriter({ + streamId: openResult.streamId, + writeChunk: window.electronAPI.writeMicrophoneSidecarStreamChunk, + }); + + return { + write: writer.write, + close: async (videoPath, options) => { + try { + await writer.waitForWrites(); + return window.electronAPI.closeMicrophoneSidecarStream( + openResult.streamId!, + videoPath, + options, + ); + } catch (error) { + await window.electronAPI.closeMicrophoneSidecarStream(openResult.streamId!, "", { + abort: true, + }); + throw error; + } + }, + abort: async () => { + await window.electronAPI.closeMicrophoneSidecarStream(openResult.streamId!, "", { + abort: true, + }); + }, + }; +} const LINUX_PORTAL_SOURCE: ProcessedDesktopSource = { id: "screen:linux-portal", name: "Linux Portal", @@ -280,6 +424,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const mixingContext = useRef(null); const chunks = useRef([]); const webcamChunks = useRef([]); + const recordingStreamWriter = useRef(null); + const webcamStreamWriter = useRef(null); const startTime = useRef(0); const webcamStartTime = useRef(null); const webcamTimeOffsetMs = useRef(0); @@ -299,6 +445,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const pauseStartedAtMs = useRef(null); const micFallbackRecorder = useRef(null); const micFallbackChunks = useRef([]); + const micFallbackStreamWriter = useRef(null); const micFallbackStartDelayMs = useRef(null); const micFallbackTrackSettings = useRef(null); const micFallbackRequestedConstraints = useRef(null); @@ -314,12 +461,35 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ); const requestedBrowserMicrophoneProfile = useRef(null); const hideEditorOverlayCursorByDefault = useRef(false); + const discardRecording = useRef(false); const notifyRecordingFinalizationFailure = useCallback(async (message: string) => { setFinalizing(false); toast.error(message, { duration: 10000 }); }, []); + const abortOpenChunkStreams = useCallback(() => { + const abortWriter = ( + writer: { abort: () => Promise } | null, + label: string, + ) => { + if (!writer) { + return; + } + + void writer.abort().catch((error) => { + console.warn(`Failed to abort ${label} stream:`, error); + }); + }; + + abortWriter(recordingStreamWriter.current, "recording"); + abortWriter(webcamStreamWriter.current, "webcam"); + abortWriter(micFallbackStreamWriter.current, "microphone sidecar"); + recordingStreamWriter.current = null; + webcamStreamWriter.current = null; + micFallbackStreamWriter.current = null; + }, []); + const logNativeCaptureDiagnostics = useCallback(async (context: string) => { if (typeof window.electronAPI?.getLastNativeCaptureDiagnostics !== "function") { return; @@ -476,10 +646,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return selectWebcamRecordingMimeType(); }, []); - const computeBitrate = (width: number, height: number) => { + const computeBitrate = (width: number, height: number, frameRate = TARGET_FRAME_RATE) => { const pixels = width * height; + const normalizedFrameRate = Number.isFinite(frameRate) ? frameRate : TARGET_FRAME_RATE; const highFrameRateBoost = - TARGET_FRAME_RATE >= HIGH_FRAME_RATE_THRESHOLD ? HIGH_FRAME_RATE_BOOST : 1; + normalizedFrameRate >= HIGH_FRAME_RATE_THRESHOLD + ? Math.min(HIGH_FRAME_RATE_BOOST, normalizedFrameRate / MIN_FRAME_RATE) + : 1; if (pixels >= FOUR_K_PIXELS) { return Math.round(BITRATE_4K * highFrameRateBoost); @@ -543,7 +716,12 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } - micFallbackChunks.current.push(event.data); + const streamWriter = micFallbackStreamWriter.current; + if (streamWriter) { + streamWriter.write(event.data); + } else { + micFallbackChunks.current.push(event.data); + } const startedAt = micFallbackRecorderStartedAt.current; if (startedAt === null) { return; @@ -688,8 +866,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { closeMicFallbackPauseInterval(); recorder.ondataavailable = appendMicFallbackChunk; recorder.onstop = () => { - const blob = - micFallbackChunks.current.length > 0 + const blob = micFallbackStreamWriter.current + ? null + : micFallbackChunks.current.length > 0 ? new Blob(micFallbackChunks.current, { type: recorder.mimeType }) : null; micFallbackChunks.current = []; @@ -741,55 +920,66 @@ export function useScreenRecorder(): UseScreenRecorderReturn { mediaTrackSettings?: MicrophoneTrackSettingsSnapshot | null, ) => { const micFallbackBlob = await micFallbackBlobPromise; - if (!micFallbackBlob) { - micFallbackStartDelayMs.current = null; - micFallbackTrackSettings.current = null; - micFallbackRequestedConstraints.current = null; - micFallbackAudioInputDevices.current = null; - micFallbackRecorderMetadata.current = null; - resetMicFallbackTimingDiagnostics(); - return; - } + const effectiveStartDelayMs = startDelayMs ?? micFallbackStartDelayMs.current; + const effectiveTrackSettings = + mediaTrackSettings ?? micFallbackTrackSettings.current; + const sidecarOptions: MicrophoneSidecarOptions = { + ...(Number.isFinite(effectiveStartDelayMs) && (effectiveStartDelayMs ?? 0) >= 0 + ? { startDelayMs: effectiveStartDelayMs ?? 0 } + : {}), + browserMicrophoneProfile: browserMicrophoneProfile.current, + ...(requestedBrowserMicrophoneProfile.current + ? { + requestedBrowserMicrophoneProfile: + requestedBrowserMicrophoneProfile.current, + } + : {}), + ...(micFallbackRequestedConstraints.current + ? { requestedConstraints: micFallbackRequestedConstraints.current } + : {}), + ...(effectiveTrackSettings + ? { mediaTrackSettings: effectiveTrackSettings } + : {}), + ...(micFallbackAudioInputDevices.current + ? { audioInputDevices: micFallbackAudioInputDevices.current } + : {}), + ...(micFallbackRecorderMetadata.current + ? { mediaRecorder: micFallbackRecorderMetadata.current } + : {}), + ...(micFallbackChunkEvents.current.length > 0 + ? { chunkEvents: [...micFallbackChunkEvents.current] } + : {}), + ...(micFallbackPauseIntervals.current.length > 0 + ? { + pauseIntervals: micFallbackPauseIntervals.current.map((interval) => ({ + ...interval, + })), + } + : {}), + }; try { + const streamWriter = micFallbackStreamWriter.current; + if (streamWriter) { + micFallbackStreamWriter.current = null; + const result = await streamWriter.close(finalPath, sidecarOptions); + if (!result.success) { + const errorMessage = + result.error || "Failed to save the fallback microphone audio track"; + console.warn("Failed to store microphone sidecar:", errorMessage); + toast.error( + `${errorMessage}. Recording was saved without the fallback microphone track.`, + { id: MICROPHONE_SIDECAR_ERROR_TOAST_ID, duration: 10000 }, + ); + } + return; + } + + if (!micFallbackBlob) { + return; + } + const arrayBuffer = await micFallbackBlob.arrayBuffer(); - const effectiveStartDelayMs = startDelayMs ?? micFallbackStartDelayMs.current; - const effectiveTrackSettings = - mediaTrackSettings ?? micFallbackTrackSettings.current; - const sidecarOptions: MicrophoneSidecarOptions = { - ...(Number.isFinite(effectiveStartDelayMs) && (effectiveStartDelayMs ?? 0) >= 0 - ? { startDelayMs: effectiveStartDelayMs ?? 0 } - : {}), - browserMicrophoneProfile: browserMicrophoneProfile.current, - ...(requestedBrowserMicrophoneProfile.current - ? { - requestedBrowserMicrophoneProfile: - requestedBrowserMicrophoneProfile.current, - } - : {}), - ...(micFallbackRequestedConstraints.current - ? { requestedConstraints: micFallbackRequestedConstraints.current } - : {}), - ...(effectiveTrackSettings - ? { mediaTrackSettings: effectiveTrackSettings } - : {}), - ...(micFallbackAudioInputDevices.current - ? { audioInputDevices: micFallbackAudioInputDevices.current } - : {}), - ...(micFallbackRecorderMetadata.current - ? { mediaRecorder: micFallbackRecorderMetadata.current } - : {}), - ...(micFallbackChunkEvents.current.length > 0 - ? { chunkEvents: [...micFallbackChunkEvents.current] } - : {}), - ...(micFallbackPauseIntervals.current.length > 0 - ? { - pauseIntervals: micFallbackPauseIntervals.current.map( - (interval) => ({ ...interval }), - ), - } - : {}), - }; const result = await window.electronAPI.storeMicrophoneSidecar( arrayBuffer, finalPath, @@ -914,7 +1104,15 @@ export function useScreenRecorder(): UseScreenRecorderReturn { }); const mimeType = selectWebcamMimeType(); + const sessionTimestamp = recordingSessionTimestamp.current ?? Date.now(); + const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${getVideoExtensionForMimeType(mimeType)}`; webcamChunks.current = []; + webcamStreamWriter.current = await createRecordingStreamWriter(webcamFileName).catch( + (error) => { + console.warn("Webcam recording stream unavailable:", error); + return null; + }, + ); resolvedWebcamPath.current = null; webcamStopPromise.current = new Promise((resolve) => { webcamStopResolver.current = resolve; @@ -928,7 +1126,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamRecorder.current = recorder; recorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0) { + if (!event.data || event.data.size <= 0 || discardRecording.current) { + return; + } + + if (webcamStreamWriter.current) { + webcamStreamWriter.current.write(event.data); + } else { webcamChunks.current.push(event.data); } }; @@ -937,12 +1141,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { webcamStopResolver.current = null; }; recorder.onstop = async () => { - const sessionTimestamp = recordingSessionTimestamp.current ?? Date.now(); const webcamMimeType = recorder.mimeType || mimeType; - const webcamFileName = `${RECORDING_FILE_PREFIX}${sessionTimestamp}${WEBCAM_SUFFIX}${getVideoExtensionForMimeType(webcamMimeType)}`; + const streamWriter = webcamStreamWriter.current; + webcamStreamWriter.current = null; try { - if (webcamChunks.current.length === 0) { + if (discardRecording.current) { + webcamChunks.current = []; webcamStopResolver.current?.(null); return; } @@ -951,6 +1156,26 @@ export function useScreenRecorder(): UseScreenRecorderReturn { 0, getRecordingDurationMs(Date.now()) - webcamTimeOffsetMs.current, ); + + if (streamWriter) { + const result = await streamWriter.close({ + mimeType: webcamMimeType, + }); + if (!result.success) { + console.warn( + "Failed to store webcam recording:", + result.error ?? result.message, + ); + } + webcamStopResolver.current?.(result.success ? (result.path ?? null) : null); + return; + } + + if (webcamChunks.current.length === 0) { + webcamStopResolver.current?.(null); + return; + } + const webcamBlob = new Blob( webcamChunks.current, webcamMimeType ? { type: webcamMimeType } : undefined, @@ -979,6 +1204,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } }; } catch (error) { + const streamWriter = webcamStreamWriter.current; + webcamStreamWriter.current = null; + void streamWriter?.abort().catch(() => undefined); console.warn( "Failed to start webcam recording; continuing without webcam layer:", error, @@ -1270,6 +1498,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cleanup?.(); removeRecordingStateListener?.(); removeRecordingInterruptedListener?.(); + discardRecording.current = true; + abortOpenChunkStreams(); if (nativeScreenRecording.current) { nativeScreenRecording.current = false; @@ -1284,7 +1514,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { cleanupCapturedMedia(); }; - }, [cleanupCapturedMedia, recoverNativeRecordingSession]); + }, [abortOpenChunkStreams, cleanupCapturedMedia, recoverNativeRecordingSession]); const startRecording = async () => { if (startInFlight.current) { @@ -1292,6 +1522,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { } hasPromptedForReselect.current = false; + discardRecording.current = false; startInFlight.current = true; setStarting(true); @@ -1449,6 +1680,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { micFallbackAudioInputDevices.current, ); micFallbackChunks.current = []; + micFallbackStreamWriter.current = + await createMicrophoneSidecarStreamWriter().catch((error) => { + console.warn("Microphone sidecar stream unavailable:", error); + return null; + }); const recorder = new MediaRecorder(micStream, { mimeType: "audio/webm;codecs=opus", audioBitsPerSecond: AUDIO_BITRATE_VOICE, @@ -1468,6 +1704,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recorder.start(RECORDER_TIMESLICE_MS); micFallbackRecorder.current = recorder; } catch (micError) { + const streamWriter = micFallbackStreamWriter.current; + micFallbackStreamWriter.current = null; + void streamWriter?.abort().catch(() => undefined); micFallbackStartDelayMs.current = null; micFallbackTrackSettings.current = null; micFallbackRequestedConstraints.current = null; @@ -1693,7 +1932,11 @@ export function useScreenRecorder(): UseScreenRecorderReturn { width = Math.floor(width / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; height = Math.floor(height / CODEC_ALIGNMENT) * CODEC_ALIGNMENT; - const videoBitsPerSecond = computeBitrate(width, height); + const videoBitsPerSecond = computeBitrate( + width, + height, + frameRate ?? TARGET_FRAME_RATE, + ); const mimeType = selectMimeType(); console.log( @@ -1703,6 +1946,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { ); chunks.current = []; + const timestamp = recordingSessionTimestamp.current ?? Date.now(); + const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`; + recordingStreamWriter.current = await createRecordingStreamWriter(videoFileName).catch( + (error) => { + console.warn("Recording stream unavailable:", error); + return null; + }, + ); const hasAudio = stream.current.getAudioTracks().length > 0; const recorder = new MediaRecorder(stream.current, { videoBitsPerSecond, @@ -1718,43 +1969,30 @@ export function useScreenRecorder(): UseScreenRecorderReturn { mediaRecorder.current = recorder; recorder.ondataavailable = (event) => { - if (event.data && event.data.size > 0) chunks.current.push(event.data); + if (!event.data || event.data.size <= 0 || discardRecording.current) { + return; + } + + if (recordingStreamWriter.current) { + recordingStreamWriter.current.write(event.data); + } else { + chunks.current.push(event.data); + } }; recorder.onstop = async () => { cleanupCapturedMedia(); - if (chunks.current.length === 0) { + const streamWriter = recordingStreamWriter.current; + recordingStreamWriter.current = null; + + if (discardRecording.current) { + chunks.current = []; + discardRecording.current = false; setFinalizing(false); return; } - const duration = getRecordingDurationMs(Date.now()); - const recordedChunks = chunks.current; - const recordingBlobType = recorder.mimeType || mimeType; - const buggyBlob = new Blob( - recordedChunks, - recordingBlobType ? { type: recordingBlobType } : undefined, - ); - chunks.current = []; - const timestamp = recordingSessionTimestamp.current ?? Date.now(); - const videoFileName = `${RECORDING_FILE_PREFIX}${timestamp}${VIDEO_FILE_EXTENSION}`; - try { - const videoBlob = await fixWebmDuration(buggyBlob, duration); - const arrayBuffer = await videoBlob.arrayBuffer(); - const videoResult = await window.electronAPI.storeRecordedVideo( - arrayBuffer, - videoFileName, - ); - if (!videoResult.success) { - console.error("Failed to store video:", videoResult.message); - await notifyRecordingFinalizationFailure( - videoResult.message || "Failed to store the recording.", - ); - return; - } - - if (videoResult.path) { - const finalVideoPath = videoResult.path; + const finalizeStoredBrowserVideo = async (finalVideoPath: string) => { // 1. Launch editor immediately (Optimistic UI) await finalizeRecordingSession(finalVideoPath, null); @@ -1777,11 +2015,63 @@ export function useScreenRecorder(): UseScreenRecorderReturn { // After all background tasks are done (webcam), // we can safely close the HUD window to release hardware and resources. if (typeof window.electronAPI?.hudOverlayClose === "function") { - console.log("[useScreenRecorder:browser] All background tasks finished, closing HUD"); + console.log( + "[useScreenRecorder:browser] All background tasks finished, closing HUD", + ); window.electronAPI.hudOverlayClose(); } } })(); + }; + const duration = getRecordingDurationMs(Date.now()); + const recordingBlobType = recorder.mimeType || mimeType; + + if (streamWriter) { + const videoResult = await streamWriter.close({ + mimeType: recordingBlobType, + }); + if (!videoResult.success || !videoResult.path) { + console.error( + "Failed to store video:", + videoResult.error ?? videoResult.message, + ); + await notifyRecordingFinalizationFailure( + videoResult.message || videoResult.error || "Failed to store the recording.", + ); + return; + } + + await finalizeStoredBrowserVideo(videoResult.path); + return; + } + + if (chunks.current.length === 0) { + setFinalizing(false); + return; + } + + const recordedChunks = chunks.current; + const buggyBlob = new Blob( + recordedChunks, + recordingBlobType ? { type: recordingBlobType } : undefined, + ); + chunks.current = []; + const videoBlob = await fixWebmDuration(buggyBlob, duration); + const arrayBuffer = await videoBlob.arrayBuffer(); + const videoResult = await window.electronAPI.storeRecordedVideo( + arrayBuffer, + videoFileName, + ); + if (!videoResult.success) { + console.error("Failed to store video:", videoResult.message); + await notifyRecordingFinalizationFailure( + videoResult.message || "Failed to store the recording.", + ); + return; + } + + if (videoResult.path) { + await finalizeStoredBrowserVideo(videoResult.path); } else { await notifyRecordingFinalizationFailure("Failed to save the recording."); } @@ -1805,6 +2095,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(true); window.electronAPI?.setRecordingState(true); } catch (error) { + discardRecording.current = true; + abortOpenChunkStreams(); console.error("Failed to start recording:", error); alert( error instanceof Error @@ -1920,6 +2212,8 @@ export function useScreenRecorder(): UseScreenRecorderReturn { const cancelRecording = useCallback(() => { if (!recording) return; + discardRecording.current = true; + abortOpenChunkStreams(); setPaused(false); markRecordingResumed(Date.now()); @@ -1963,7 +2257,7 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(false); window.electronAPI?.setRecordingState(false); } - }, [cleanupCapturedMedia, markRecordingResumed, recording]); + }, [abortOpenChunkStreams, cleanupCapturedMedia, markRecordingResumed, recording]); const toggleRecording = async () => { if (starting || countdownActive || finalizing) {