add: decoding frame in backend server

src/electron/backend/consts.ts

@@ -0,0 +1,3 @@
+export const RECORD_FRAME_RATE = 0.5;
+export const ENCODING_FRAME_RATE = 30;
+export const ENCODING_FRAME_INTERVAL = 1 / ENCODING_FRAME_RATE;

src/electron/backend/encoding.ts

@@ -6,13 +6,13 @@ import type { EncodingTask, Frame } from "./schema";
 import sizeOf from "image-size";
 import {
 	getEncodingTempDir,
-	getFFmpegCommand,
+	getEncodeCommand,
 	getRecordingsDir,
 	getScreenshotsDir
 } from "../utils/backend.js";
 import cache from "memory-cache";
+import { ENCODING_FRAME_INTERVAL, RECORD_FRAME_RATE as FRAME_RATE } from "./consts.js";
 
-const FRAME_RATE = 0.5;
 const THREE_MINUTES = 180;
 const MIN_FRAMES_TO_ENCODE = THREE_MINUTES * FRAME_RATE;
 const CONCURRENCY = 1;
@@ -22,7 +22,7 @@ export function checkFramesForEncoding(db: Database) {
 	const stmt = db.prepare(`
         SELECT id, imgFilename, createdAt
         FROM frame
-        WHERE encodeStatus = 0
+        WHERE encodeStatus = 0 AND imgFilename IS NOT NULL
         ORDER BY createdAt ASC;
     `);
 	const frames = stmt.all() as Frame[];
@@ -32,16 +32,16 @@ export function checkFramesForEncoding(db: Database) {
 	for (let i = 1; i < frames.length; i++) {
 		const frame = frames[i];
 		const lastFrame = frames[i - 1];
-		const framePath = join(getScreenshotsDir(), frame.imgFilename);
-		const lastFramePath = join(getScreenshotsDir(), lastFrame.imgFilename);
+		const framePath = join(getScreenshotsDir(), frame.imgFilename!);
+		const lastFramePath = join(getScreenshotsDir(), lastFrame.imgFilename!);
 		if (!fs.existsSync(framePath)) {
 			console.warn("File not exist:", frame.imgFilename);
-			deleteFrameFromDB(db, frame.id)
+			deleteFrameFromDB(db, frame.id);
 			continue;
 		}
 		if (!fs.existsSync(lastFramePath)) {
 			console.warn("File not exist:", lastFrame.imgFilename);
-			deleteFrameFromDB(db, lastFrame.id)
+			deleteFrameFromDB(db, lastFrame.id);
 			continue;
 		}
 		const currentFrameSize = sizeOf(framePath);
@@ -78,11 +78,13 @@ export function checkFramesForEncoding(db: Database) {
 }
 
 function deleteEncodedScreenshots(db: Database) {
+	// TODO: double-check that the frame was really encoded into the video
 	const stmt = db.prepare(`
 	    SELECT * FROM frame WHERE encodeStatus = 2 AND imgFilename IS NOT NULL;
 	`);
 	const frames = stmt.all() as Frame[];
 	for (const frame of frames) {
+		if (!frame.imgFilename) continue;
 		fs.unlinkSync(path.join(getScreenshotsDir(), frame.imgFilename));
 		const updateStmt = db.prepare(`
 			UPDATE frame SET imgFilename = NULL WHERE id = ?;
@@ -91,7 +93,7 @@ function deleteEncodedScreenshots(db: Database) {
 	}
 }
 
-function deleteNonExistentScreenshots(db: Database) {
+function _deleteNonExistentScreenshots(db: Database) {
 	const screenshotDir = getScreenshotsDir();
 	const filesInDir = new Set(fs.readdirSync(screenshotDir));
 
@@ -128,10 +130,12 @@ function getTasksPerforming() {
 
 function createMetaFile(frames: Frame[]) {
 	return frames
-		.map(
-			(frame) =>
-				`file '${path.join(getScreenshotsDir(), frame.imgFilename)}'\nduration 0.03333`
-		)
+		.map((frame) => {
+			if (!frame.imgFilename) return "";
+			const framePath = join(getScreenshotsDir(), frame.imgFilename);
+			const duration = ENCODING_FRAME_INTERVAL.toFixed(5);
+			return `file '${framePath}'\nduration ${duration}`;
+		})
 		.join("\n");
 }
 
@@ -175,7 +179,7 @@ export function processEncodingTasks(db: Database) {
 		cache.put("backend:encodingTasksPerforming", [...tasksPerforming, taskId.toString()]);
 
 		const videoPath = path.join(getRecordingsDir(), `${taskId}.mp4`);
-		const ffmpegCommand = getFFmpegCommand(metaFilePath, videoPath);
+		const ffmpegCommand = getEncodeCommand(metaFilePath, videoPath);
 		console.log("FFMPEG", ffmpegCommand);
 		exec(ffmpegCommand, (error, _stdout, _stderr) => {
 			if (error) {

src/electron/backend/schema.d.ts

@@ -1,7 +1,7 @@
 export interface Frame {
 	id: number;
 	createdAt: number;
-	imgFilename: string;
+	imgFilename: string | null;
 	segmentID: number | null;
 	videoPath: string | null;
 	videoFrameIndex: number | null;

src/electron/index.ts

@@ -26,7 +26,7 @@ import honoApp from "./server/index.js";
 import { serve } from "@hono/node-server";
 import { findAvailablePort } from "./utils/server.js";
 import cache from "memory-cache";
-import { generate } from "@alikia/random-key";
+import { generate as generateAPIKey } from "@alikia/random-key";
 
 const i18n = initI18n();
 
@@ -93,10 +93,11 @@ app.on("activate", () => {});
 app.on("ready", () => {
 	createTray();
 	findAvailablePort(12412).then((port) => {
-		generate().then((key) => {
-			cache.put("server:APIKey", key);
+		generateAPIKey().then((key) => {
 			cache.put("server:port", port);
-			if (dev) console.log(`API Key: ${key}`);
+			if (!dev) {
+				cache.put("server:APIKey", key);
+			}
 			serve({ fetch: honoApp.fetch, port: port });
 			console.log(`App server running on port ${port}`);
 		});
@@ -122,10 +123,6 @@ app.on("will-quit", () => {
 	if (screenshotInterval) clearInterval(screenshotInterval);
 });
 
-// app.on("window-all-closed", () => {
-// 	if (process.platform !== "darwin") app.quit();
-// });
-
 ipcMain.on("close-settings", () => {
 	settingsWindow?.hide();
 });

src/electron/server/index.ts

@@ -4,7 +4,15 @@ import { join } from "path";
 import fs from "fs";
 import { Database } from "better-sqlite3";
 import type { Frame } from "../backend/schema";
-import { getScreenshotsDir } from "../utils/backend.js";
+import {
+	extractFramesFromVideo,
+	getDecodingTempDir,
+	getRecordingsDir,
+	getScreenshotsDir,
+	immediatelyExtractFrameFromVideo,
+	waitForFileExists
+} from "../utils/backend.js";
+import { existsSync } from "fs";
 
 const app = new Hono();
 
@@ -64,32 +72,77 @@ app.get("/frame/:id", async (c) => {
 	const frame = db
 		.prepare(
 			`
-    SELECT imgFilename, videoPath, videoFrameIndex 
-    FROM frame 
-    WHERE id = ?
-  `
+				SELECT imgFilename, videoPath, videoFrameIndex, createdAt
+				FROM frame 
+				WHERE id = ?
+			`
 		)
-		.get(id) as Frame;
+		.get(id) as Frame | undefined;
 
-	if (!frame) {
-		return c.json({ error: "Frame not found" }, 404);
+	if (!frame) return c.json({ error: "Frame not found" }, 404);
+	
+	const decodingTempDir = getDecodingTempDir();
+	const screenshotsDir = getScreenshotsDir();
+	const videoFilename = frame.videoPath;
+	const frameIndex = frame.videoFrameIndex;
+	const imageFilename = frame.imgFilename;
+	const bareVideoFilename = videoFilename?.replace(".mp4", "") || null;
+	const decodedImageBMP = frameIndex
+		? `${bareVideoFilename}_${(frameIndex).toString().padStart(4, "0")}.bmp`
+		: null;
+	const decodedImagePNG = frameIndex
+		? `${bareVideoFilename}_${(frameIndex).toString().padStart(4, "0")}.png`
+		: null;
+	let returnImagePath = "";
+
+	let needToBeDecoded = videoFilename !== null && frameIndex !== null && !frame.imgFilename;
+	if (decodedImagePNG && fs.existsSync(join(getDecodingTempDir(), decodedImagePNG))) {
+		needToBeDecoded = false;
+		returnImagePath = join(decodingTempDir, decodedImagePNG);
+	} else if (decodedImageBMP && fs.existsSync(join(getDecodingTempDir(), decodedImageBMP))) {
+		needToBeDecoded = false;
+		returnImagePath = join(decodingTempDir, decodedImageBMP);
+	}
+	else if (imageFilename && fs.existsSync(join(screenshotsDir, imageFilename))) {
+		returnImagePath = join(screenshotsDir, imageFilename);
 	}
 
-	// If frame is from video, decode and return frame
-	if (frame.videoPath) {
-		// TODO: Implement video frame extraction
-		return c.json({ error: "Video frame extraction not implemented" }, 501);
-	}
+	if (needToBeDecoded) {
+		const videoExists = fs.existsSync(join(getRecordingsDir(), videoFilename!));
 
-	// Return image file
-	const imagePath = join(getScreenshotsDir(), frame.imgFilename);
-	const imageBuffer = fs.readFileSync(imagePath);
-	return new Response(imageBuffer, {
-		status: 200,
-		headers: {
-			"Content-Type": "image/png"
+		if (!videoExists) {
+			return c.json({ error: "Video not found" }, { status: 404 });
 		}
-	});
+		
+		// Decode requesting frame immediately, then decode the whole video chunk
+		// This allows the user to get an immediate response,
+		// and we get prepared for the next few seconds of scrolling (timeline).
+		const decodedFilename = immediatelyExtractFrameFromVideo(videoFilename!, frameIndex!, decodingTempDir);
+		const decodedPath = join(decodingTempDir, decodedFilename);
+
+		await waitForFileExists(decodedPath);
+		extractFramesFromVideo(videoFilename!, null, null, decodingTempDir);
+
+		if (existsSync(decodedPath)) {
+			const imageBuffer = fs.readFileSync(decodedPath);
+			return new Response(imageBuffer, {
+				status: 200,
+				headers: {
+					"Content-Type": "image/bmp"
+				}
+			});
+		} else {
+			return c.json({ error: "Frame cannot be decoded" }, { status: 500 });
+		}
+	} else {
+		const imageBuffer = fs.readFileSync(returnImagePath);
+		return new Response(imageBuffer, {
+			status: 200,
+			headers: {
+				"Content-Type": "image/png"
+			}
+		});
+	}
 });
 
 export default app;

src/electron/utils/backend.ts

@@ -1,9 +1,17 @@
-import path from "path";
+import path, { join } from "path";
 import os from "os";
 import fs from "fs";
 import { __dirname } from "../dirname.js";
-import { execSync } from "child_process";
+import { execSync, spawn } from "child_process";
 import cache from "memory-cache";
+import { exec } from "child_process";
+import { ENCODING_FRAME_RATE } from "../backend/consts.js";
+
+const DECODE_CONCURRENCY = 1;
+
+function getTasksPerforming() {
+	return (cache.get("backend:extractFramesTasksPerforming") as string[]) || [];
+}
 
 export function getUserDataDir() {
 	switch (process.platform) {
@@ -72,6 +80,15 @@ export function getEncodingTempDir() {
 	return encodingTempDir;
 }
 
+export function getDecodingTempDir() {
+	const tempDir = createTempDir();
+	const decodingTempDir = path.join(tempDir, "decoding");
+	if (!fs.existsSync(decodingTempDir)) {
+		fs.mkdirSync(decodingTempDir, { recursive: true });
+	}
+	return decodingTempDir;
+}
+
 export function getFFmpegPath() {
 	switch (process.platform) {
 		case "win32":
@@ -101,7 +118,164 @@ function getBestCodec() {
 	return codec;
 }
 
-export function getFFmpegCommand(metaFilePath: string, videoPath: string) {
+export function getEncodeCommand(metaFilePath: string, videoPath: string) {
 	const codec = getBestCodec();
-	return `${getFFmpegPath()} -f concat -safe 0 -i "${metaFilePath}" -c:v ${codec} -r 30 -y -threads 1 "${videoPath}"`;
+	return `${getFFmpegPath()} -f concat -safe 0 -i "${metaFilePath}" -c:v ${codec} -r ${ENCODING_FRAME_RATE} -y -threads 1 "${videoPath}"`;
+}
+
+/**
+ * Extracts frames from a video file using FFmpeg
+ *
+ * @async
+ * @param {string} videoFilename - The name of the video file to extract frames from
+ * @param {number|null} startIndex - The starting frame index (inclusive). If null, starts from first frame
+ * @param {number|null} endIndex - The ending frame index (inclusive). If null, goes to last frame
+ * @param {string|null} outputPath - The output path for the extracted frames
+ */
+export async function extractFramesFromVideo(
+	videoFilename: string,
+	startIndex: number | null,
+	endIndex: number | null,
+	outputPath: string = ".",
+	format: "png" | "bmp" = "png"
+) {
+	const tasksPerforming = getTasksPerforming();
+	if (tasksPerforming.length >= DECODE_CONCURRENCY) {
+		console.log(`Reached concurrency limit (${DECODE_CONCURRENCY}), skipping extraction`);
+		return;
+	}
+
+	const taskId = `${videoFilename}-${startIndex}-${endIndex}`;
+	cache.put("backend:extractFramesTasksPerforming", [...tasksPerforming, taskId]);
+	const fullVideoPath = join(getRecordingsDir(), videoFilename);
+
+	const beginTimeArg =
+		startIndex !== null ? `-ss ${formatTime(startIndex / ENCODING_FRAME_RATE)}` : "";
+	const endTimeArg = endIndex !== null ? `-to ${formatTime(endIndex / ENCODING_FRAME_RATE)}` : "";
+
+	let videoFilter = "";
+	if (startIndex !== null && endIndex !== null) {
+		videoFilter = `select='between(n\\,${startIndex}\\,${endIndex})'`;
+	} else if (startIndex !== null) {
+		videoFilter = `select='gte(n\\,${startIndex})'`;
+	} else if (endIndex !== null) {
+		videoFilter = `select='lte(n\\,${endIndex})'`;
+	}
+
+	const bareVideoFilename = videoFilename.split(".").slice(0, -1).join(".");
+	const outputPathArg = join(outputPath, `${bareVideoFilename}_%04d.${format}`);
+
+	const command = [
+		getFFmpegPath(),
+		beginTimeArg,
+		endTimeArg,
+		`-i "${fullVideoPath}"`,
+		videoFilter ? `-vf ${videoFilter}` : "",
+		`-start_number ${startIndex || 0}`,
+		`"${outputPathArg}"`
+	]
+		.filter((arg) => arg !== "")
+		.join(" ");
+
+	exec(command, (error, _stdout, _stderr) => {
+		if (error) {
+			console.error(`FFmpeg error: ${error.message}`);
+		}
+		const tasksPerforming = getTasksPerforming();
+		cache.put(
+			"backend:extractFramesTasksPerforming",
+			tasksPerforming.filter((id) => id !== taskId)
+		);
+	});
+}
+
+export function immediatelyExtractFrameFromVideo(
+	videoFilename: string,
+	frameIndex: number,
+	outputPath = "."
+) {
+	const bareVideoFilename = videoFilename.split(".").slice(0, -1).join(".");
+	const fullVideoPath = join(getRecordingsDir(), videoFilename);
+	const outputFilename = `${bareVideoFilename}_${frameIndex.toString().padStart(4, "0")}.bmp`;
+	const outputPathArg = join(outputPath, outputFilename);
+	const args = [
+		"-i",
+		`${fullVideoPath}`,
+		"-ss",
+		`${formatTime(frameIndex / ENCODING_FRAME_RATE)}`,
+		"-vframes",
+		"1",
+		`${outputPathArg}`
+	];
+	const ffmpeg = spawn("ffmpeg", args);
+	ffmpeg.stdout.on("data", (data) => {
+		console.log(data.toString());
+	});
+
+	ffmpeg.stderr.on("data", (data) => {
+		console.log(data.toString());
+	});
+	ffmpeg.on("exit", (code) => {
+		if (code !== 0) {
+			console.error("Error extracting frame:", code);
+		}
+	});
+	return outputFilename;
+}
+
+function formatTime(seconds: number): string {
+	// Calculate hours, minutes, seconds, and milliseconds
+	const hours = Math.floor(seconds / 3600);
+	const minutes = Math.floor((seconds % 3600) / 60);
+	const secs = Math.floor(seconds % 60);
+	const milliseconds = Math.round((seconds % 1) * 1000);
+
+	// Format the output with leading zeros
+	const formattedTime =
+		[
+			String(hours).padStart(2, "0"),
+			String(minutes).padStart(2, "0"),
+			String(secs).padStart(2, "0")
+		].join(":") +
+		"." +
+		String(milliseconds).padStart(3, "0");
+
+	return formattedTime;
+}
+
+export async function waitForFileExists(filePath: string, timeout: number = 10000): Promise<void> {
+	return new Promise((resolve, reject) => {
+		fs.access(filePath, fs.constants.F_OK, (err) => {
+			if (!err) {
+				resolve();
+				return;
+			}
+
+			const dir = path.dirname(filePath);
+			const filename = path.basename(filePath);
+
+			const watcher = fs.watch(dir, (eventType, watchedFilename) => {
+				if (eventType === "rename" && watchedFilename === filename) {
+					fs.access(filePath, fs.constants.F_OK, (err) => {
+						if (!err) {
+							clearTimeout(timeoutId);
+							watcher.close();
+							resolve();
+						}
+					});
+				}
+			});
+
+			watcher.on("error", (err) => {
+				clearTimeout(timeoutId);
+				watcher.close();
+				reject(err);
+			});
+
+			const timeoutId = setTimeout(() => {
+				watcher.close();
+				reject(new Error(`Timeout: File ${filePath} did not exist within ${timeout}ms`));
+			}, timeout);
+		});
+	});
 }