From 97dce812977dc327bf9256bf4e62da507d77b8c1 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Wed, 1 Jan 2025 02:22:14 +0800 Subject: [PATCH] add: decoding frame in backend server --- src/electron/backend/consts.ts | 3 + src/electron/backend/encoding.ts | 30 ++--- src/electron/backend/schema.d.ts | 2 +- src/electron/index.ts | 13 +-- src/electron/server/index.ts | 95 ++++++++++++---- src/electron/utils/backend.ts | 182 ++++++++++++++++++++++++++++++- 6 files changed, 278 insertions(+), 47 deletions(-) create mode 100644 src/electron/backend/consts.ts diff --git a/src/electron/backend/consts.ts b/src/electron/backend/consts.ts new file mode 100644 index 0000000..801f0ad --- /dev/null +++ b/src/electron/backend/consts.ts @@ -0,0 +1,3 @@ +export const RECORD_FRAME_RATE = 0.5; +export const ENCODING_FRAME_RATE = 30; +export const ENCODING_FRAME_INTERVAL = 1 / ENCODING_FRAME_RATE; \ No newline at end of file diff --git a/src/electron/backend/encoding.ts b/src/electron/backend/encoding.ts index 527f734..01976f8 100644 --- a/src/electron/backend/encoding.ts +++ b/src/electron/backend/encoding.ts @@ -6,13 +6,13 @@ import type { EncodingTask, Frame } from "./schema"; import sizeOf from "image-size"; import { getEncodingTempDir, - getFFmpegCommand, + getEncodeCommand, getRecordingsDir, getScreenshotsDir } from "../utils/backend.js"; import cache from "memory-cache"; +import { ENCODING_FRAME_INTERVAL, RECORD_FRAME_RATE as FRAME_RATE } from "./consts.js"; -const FRAME_RATE = 0.5; const THREE_MINUTES = 180; const MIN_FRAMES_TO_ENCODE = THREE_MINUTES * FRAME_RATE; const CONCURRENCY = 1; @@ -22,7 +22,7 @@ export function checkFramesForEncoding(db: Database) { const stmt = db.prepare(` SELECT id, imgFilename, createdAt FROM frame - WHERE encodeStatus = 0 + WHERE encodeStatus = 0 AND imgFilename IS NOT NULL ORDER BY createdAt ASC; `); const frames = stmt.all() as Frame[]; @@ -32,16 +32,16 @@ export function checkFramesForEncoding(db: Database) { for (let i = 1; i < frames.length; i++) { const frame = frames[i]; const lastFrame = frames[i - 1]; - const framePath = join(getScreenshotsDir(), frame.imgFilename); - const lastFramePath = join(getScreenshotsDir(), lastFrame.imgFilename); + const framePath = join(getScreenshotsDir(), frame.imgFilename!); + const lastFramePath = join(getScreenshotsDir(), lastFrame.imgFilename!); if (!fs.existsSync(framePath)) { console.warn("File not exist:", frame.imgFilename); - deleteFrameFromDB(db, frame.id) + deleteFrameFromDB(db, frame.id); continue; } if (!fs.existsSync(lastFramePath)) { console.warn("File not exist:", lastFrame.imgFilename); - deleteFrameFromDB(db, lastFrame.id) + deleteFrameFromDB(db, lastFrame.id); continue; } const currentFrameSize = sizeOf(framePath); @@ -78,11 +78,13 @@ export function checkFramesForEncoding(db: Database) { } function deleteEncodedScreenshots(db: Database) { + // TODO: double-check that the frame was really encoded into the video const stmt = db.prepare(` SELECT * FROM frame WHERE encodeStatus = 2 AND imgFilename IS NOT NULL; `); const frames = stmt.all() as Frame[]; for (const frame of frames) { + if (!frame.imgFilename) continue; fs.unlinkSync(path.join(getScreenshotsDir(), frame.imgFilename)); const updateStmt = db.prepare(` UPDATE frame SET imgFilename = NULL WHERE id = ?; @@ -91,7 +93,7 @@ function deleteEncodedScreenshots(db: Database) { } } -function deleteNonExistentScreenshots(db: Database) { +function _deleteNonExistentScreenshots(db: Database) { const screenshotDir = getScreenshotsDir(); const filesInDir = new Set(fs.readdirSync(screenshotDir)); @@ -128,10 +130,12 @@ function getTasksPerforming() { function createMetaFile(frames: Frame[]) { return frames - .map( - (frame) => - `file '${path.join(getScreenshotsDir(), frame.imgFilename)}'\nduration 0.03333` - ) + .map((frame) => { + if (!frame.imgFilename) return ""; + const framePath = join(getScreenshotsDir(), frame.imgFilename); + const duration = ENCODING_FRAME_INTERVAL.toFixed(5); + return `file '${framePath}'\nduration ${duration}`; + }) .join("\n"); } @@ -175,7 +179,7 @@ export function processEncodingTasks(db: Database) { cache.put("backend:encodingTasksPerforming", [...tasksPerforming, taskId.toString()]); const videoPath = path.join(getRecordingsDir(), `${taskId}.mp4`); - const ffmpegCommand = getFFmpegCommand(metaFilePath, videoPath); + const ffmpegCommand = getEncodeCommand(metaFilePath, videoPath); console.log("FFMPEG", ffmpegCommand); exec(ffmpegCommand, (error, _stdout, _stderr) => { if (error) { diff --git a/src/electron/backend/schema.d.ts b/src/electron/backend/schema.d.ts index 1c3d40f..c497173 100644 --- a/src/electron/backend/schema.d.ts +++ b/src/electron/backend/schema.d.ts @@ -1,7 +1,7 @@ export interface Frame { id: number; createdAt: number; - imgFilename: string; + imgFilename: string | null; segmentID: number | null; videoPath: string | null; videoFrameIndex: number | null; diff --git a/src/electron/index.ts b/src/electron/index.ts index 568ba77..0397f5a 100644 --- a/src/electron/index.ts +++ b/src/electron/index.ts @@ -26,7 +26,7 @@ import honoApp from "./server/index.js"; import { serve } from "@hono/node-server"; import { findAvailablePort } from "./utils/server.js"; import cache from "memory-cache"; -import { generate } from "@alikia/random-key"; +import { generate as generateAPIKey } from "@alikia/random-key"; const i18n = initI18n(); @@ -93,10 +93,11 @@ app.on("activate", () => {}); app.on("ready", () => { createTray(); findAvailablePort(12412).then((port) => { - generate().then((key) => { - cache.put("server:APIKey", key); + generateAPIKey().then((key) => { cache.put("server:port", port); - if (dev) console.log(`API Key: ${key}`); + if (!dev) { + cache.put("server:APIKey", key); + } serve({ fetch: honoApp.fetch, port: port }); console.log(`App server running on port ${port}`); }); @@ -122,10 +123,6 @@ app.on("will-quit", () => { if (screenshotInterval) clearInterval(screenshotInterval); }); -// app.on("window-all-closed", () => { -// if (process.platform !== "darwin") app.quit(); -// }); - ipcMain.on("close-settings", () => { settingsWindow?.hide(); }); diff --git a/src/electron/server/index.ts b/src/electron/server/index.ts index f54e2e4..5a20454 100644 --- a/src/electron/server/index.ts +++ b/src/electron/server/index.ts @@ -4,7 +4,15 @@ import { join } from "path"; import fs from "fs"; import { Database } from "better-sqlite3"; import type { Frame } from "../backend/schema"; -import { getScreenshotsDir } from "../utils/backend.js"; +import { + extractFramesFromVideo, + getDecodingTempDir, + getRecordingsDir, + getScreenshotsDir, + immediatelyExtractFrameFromVideo, + waitForFileExists +} from "../utils/backend.js"; +import { existsSync } from "fs"; const app = new Hono(); @@ -64,32 +72,77 @@ app.get("/frame/:id", async (c) => { const frame = db .prepare( ` - SELECT imgFilename, videoPath, videoFrameIndex - FROM frame - WHERE id = ? - ` + SELECT imgFilename, videoPath, videoFrameIndex, createdAt + FROM frame + WHERE id = ? + ` ) - .get(id) as Frame; + .get(id) as Frame | undefined; - if (!frame) { - return c.json({ error: "Frame not found" }, 404); + if (!frame) return c.json({ error: "Frame not found" }, 404); + + const decodingTempDir = getDecodingTempDir(); + const screenshotsDir = getScreenshotsDir(); + const videoFilename = frame.videoPath; + const frameIndex = frame.videoFrameIndex; + const imageFilename = frame.imgFilename; + const bareVideoFilename = videoFilename?.replace(".mp4", "") || null; + const decodedImageBMP = frameIndex + ? `${bareVideoFilename}_${(frameIndex).toString().padStart(4, "0")}.bmp` + : null; + const decodedImagePNG = frameIndex + ? `${bareVideoFilename}_${(frameIndex).toString().padStart(4, "0")}.png` + : null; + let returnImagePath = ""; + + let needToBeDecoded = videoFilename !== null && frameIndex !== null && !frame.imgFilename; + if (decodedImagePNG && fs.existsSync(join(getDecodingTempDir(), decodedImagePNG))) { + needToBeDecoded = false; + returnImagePath = join(decodingTempDir, decodedImagePNG); + } else if (decodedImageBMP && fs.existsSync(join(getDecodingTempDir(), decodedImageBMP))) { + needToBeDecoded = false; + returnImagePath = join(decodingTempDir, decodedImageBMP); + } + else if (imageFilename && fs.existsSync(join(screenshotsDir, imageFilename))) { + returnImagePath = join(screenshotsDir, imageFilename); } - // If frame is from video, decode and return frame - if (frame.videoPath) { - // TODO: Implement video frame extraction - return c.json({ error: "Video frame extraction not implemented" }, 501); - } + if (needToBeDecoded) { + const videoExists = fs.existsSync(join(getRecordingsDir(), videoFilename!)); - // Return image file - const imagePath = join(getScreenshotsDir(), frame.imgFilename); - const imageBuffer = fs.readFileSync(imagePath); - return new Response(imageBuffer, { - status: 200, - headers: { - "Content-Type": "image/png" + if (!videoExists) { + return c.json({ error: "Video not found" }, { status: 404 }); } - }); + + // Decode requesting frame immediately, then decode the whole video chunk + // This allows the user to get an immediate response, + // and we get prepared for the next few seconds of scrolling (timeline). + const decodedFilename = immediatelyExtractFrameFromVideo(videoFilename!, frameIndex!, decodingTempDir); + const decodedPath = join(decodingTempDir, decodedFilename); + + await waitForFileExists(decodedPath); + extractFramesFromVideo(videoFilename!, null, null, decodingTempDir); + + if (existsSync(decodedPath)) { + const imageBuffer = fs.readFileSync(decodedPath); + return new Response(imageBuffer, { + status: 200, + headers: { + "Content-Type": "image/bmp" + } + }); + } else { + return c.json({ error: "Frame cannot be decoded" }, { status: 500 }); + } + } else { + const imageBuffer = fs.readFileSync(returnImagePath); + return new Response(imageBuffer, { + status: 200, + headers: { + "Content-Type": "image/png" + } + }); + } }); export default app; diff --git a/src/electron/utils/backend.ts b/src/electron/utils/backend.ts index 7ac29ae..afffa85 100644 --- a/src/electron/utils/backend.ts +++ b/src/electron/utils/backend.ts @@ -1,9 +1,17 @@ -import path from "path"; +import path, { join } from "path"; import os from "os"; import fs from "fs"; import { __dirname } from "../dirname.js"; -import { execSync } from "child_process"; +import { execSync, spawn } from "child_process"; import cache from "memory-cache"; +import { exec } from "child_process"; +import { ENCODING_FRAME_RATE } from "../backend/consts.js"; + +const DECODE_CONCURRENCY = 1; + +function getTasksPerforming() { + return (cache.get("backend:extractFramesTasksPerforming") as string[]) || []; +} export function getUserDataDir() { switch (process.platform) { @@ -72,6 +80,15 @@ export function getEncodingTempDir() { return encodingTempDir; } +export function getDecodingTempDir() { + const tempDir = createTempDir(); + const decodingTempDir = path.join(tempDir, "decoding"); + if (!fs.existsSync(decodingTempDir)) { + fs.mkdirSync(decodingTempDir, { recursive: true }); + } + return decodingTempDir; +} + export function getFFmpegPath() { switch (process.platform) { case "win32": @@ -101,7 +118,164 @@ function getBestCodec() { return codec; } -export function getFFmpegCommand(metaFilePath: string, videoPath: string) { +export function getEncodeCommand(metaFilePath: string, videoPath: string) { const codec = getBestCodec(); - return `${getFFmpegPath()} -f concat -safe 0 -i "${metaFilePath}" -c:v ${codec} -r 30 -y -threads 1 "${videoPath}"`; + return `${getFFmpegPath()} -f concat -safe 0 -i "${metaFilePath}" -c:v ${codec} -r ${ENCODING_FRAME_RATE} -y -threads 1 "${videoPath}"`; +} + +/** + * Extracts frames from a video file using FFmpeg + * + * @async + * @param {string} videoFilename - The name of the video file to extract frames from + * @param {number|null} startIndex - The starting frame index (inclusive). If null, starts from first frame + * @param {number|null} endIndex - The ending frame index (inclusive). If null, goes to last frame + * @param {string|null} outputPath - The output path for the extracted frames + */ +export async function extractFramesFromVideo( + videoFilename: string, + startIndex: number | null, + endIndex: number | null, + outputPath: string = ".", + format: "png" | "bmp" = "png" +) { + const tasksPerforming = getTasksPerforming(); + if (tasksPerforming.length >= DECODE_CONCURRENCY) { + console.log(`Reached concurrency limit (${DECODE_CONCURRENCY}), skipping extraction`); + return; + } + + const taskId = `${videoFilename}-${startIndex}-${endIndex}`; + cache.put("backend:extractFramesTasksPerforming", [...tasksPerforming, taskId]); + const fullVideoPath = join(getRecordingsDir(), videoFilename); + + const beginTimeArg = + startIndex !== null ? `-ss ${formatTime(startIndex / ENCODING_FRAME_RATE)}` : ""; + const endTimeArg = endIndex !== null ? `-to ${formatTime(endIndex / ENCODING_FRAME_RATE)}` : ""; + + let videoFilter = ""; + if (startIndex !== null && endIndex !== null) { + videoFilter = `select='between(n\\,${startIndex}\\,${endIndex})'`; + } else if (startIndex !== null) { + videoFilter = `select='gte(n\\,${startIndex})'`; + } else if (endIndex !== null) { + videoFilter = `select='lte(n\\,${endIndex})'`; + } + + const bareVideoFilename = videoFilename.split(".").slice(0, -1).join("."); + const outputPathArg = join(outputPath, `${bareVideoFilename}_%04d.${format}`); + + const command = [ + getFFmpegPath(), + beginTimeArg, + endTimeArg, + `-i "${fullVideoPath}"`, + videoFilter ? `-vf ${videoFilter}` : "", + `-start_number ${startIndex || 0}`, + `"${outputPathArg}"` + ] + .filter((arg) => arg !== "") + .join(" "); + + exec(command, (error, _stdout, _stderr) => { + if (error) { + console.error(`FFmpeg error: ${error.message}`); + } + const tasksPerforming = getTasksPerforming(); + cache.put( + "backend:extractFramesTasksPerforming", + tasksPerforming.filter((id) => id !== taskId) + ); + }); +} + +export function immediatelyExtractFrameFromVideo( + videoFilename: string, + frameIndex: number, + outputPath = "." +) { + const bareVideoFilename = videoFilename.split(".").slice(0, -1).join("."); + const fullVideoPath = join(getRecordingsDir(), videoFilename); + const outputFilename = `${bareVideoFilename}_${frameIndex.toString().padStart(4, "0")}.bmp`; + const outputPathArg = join(outputPath, outputFilename); + const args = [ + "-i", + `${fullVideoPath}`, + "-ss", + `${formatTime(frameIndex / ENCODING_FRAME_RATE)}`, + "-vframes", + "1", + `${outputPathArg}` + ]; + const ffmpeg = spawn("ffmpeg", args); + ffmpeg.stdout.on("data", (data) => { + console.log(data.toString()); + }); + + ffmpeg.stderr.on("data", (data) => { + console.log(data.toString()); + }); + ffmpeg.on("exit", (code) => { + if (code !== 0) { + console.error("Error extracting frame:", code); + } + }); + return outputFilename; +} + +function formatTime(seconds: number): string { + // Calculate hours, minutes, seconds, and milliseconds + const hours = Math.floor(seconds / 3600); + const minutes = Math.floor((seconds % 3600) / 60); + const secs = Math.floor(seconds % 60); + const milliseconds = Math.round((seconds % 1) * 1000); + + // Format the output with leading zeros + const formattedTime = + [ + String(hours).padStart(2, "0"), + String(minutes).padStart(2, "0"), + String(secs).padStart(2, "0") + ].join(":") + + "." + + String(milliseconds).padStart(3, "0"); + + return formattedTime; +} + +export async function waitForFileExists(filePath: string, timeout: number = 10000): Promise { + return new Promise((resolve, reject) => { + fs.access(filePath, fs.constants.F_OK, (err) => { + if (!err) { + resolve(); + return; + } + + const dir = path.dirname(filePath); + const filename = path.basename(filePath); + + const watcher = fs.watch(dir, (eventType, watchedFilename) => { + if (eventType === "rename" && watchedFilename === filename) { + fs.access(filePath, fs.constants.F_OK, (err) => { + if (!err) { + clearTimeout(timeoutId); + watcher.close(); + resolve(); + } + }); + } + }); + + watcher.on("error", (err) => { + clearTimeout(timeoutId); + watcher.close(); + reject(err); + }); + + const timeoutId = setTimeout(() => { + watcher.close(); + reject(new Error(`Timeout: File ${filePath} did not exist within ${timeout}ms`)); + }, timeout); + }); + }); }