feature: background recording

package.json

@@ -1,6 +1,6 @@
 {
   "name": "openrewind",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "type": "module",
   "description": "Your second brain, superpowered.",
   "main": "dist/dev/index.js",
@@ -19,6 +19,7 @@
   "license": "MIT",
   "dependencies": {
     "@unly/universal-language-detector": "^2.0.3",
+    "better-sqlite3": "^11.6.0",
     "electron-context-menu": "^4.0.4",
     "electron-reloader": "^1.2.3",
     "electron-screencapture": "^1.1.0",
@@ -26,8 +27,6 @@
     "electron-store": "^10.0.0",
     "electron-window-state": "^5.0.3",
     "execa": "^9.5.1",
-    "ffmpeg-static": "^5.2.0",
-    "fluent-ffmpeg": "^2.1.3",
     "i18next": "^24.0.2",
     "i18next-browser-languagedetector": "^8.0.0",
     "i18next-electron-fs-backend": "^3.0.2",
@@ -38,15 +37,20 @@
     "react-i18next": "^15.1.2",
     "react-router": "^7.0.1",
     "react-router-dom": "^7.0.1",
+    "sqlite3": "^5.1.7",
+    "sqlstring": "^2.3.3",
     "vite-tsconfig-paths": "^5.1.3"
   },
   "devDependencies": {
+    "@electron/rebuild": "^3.7.1",
     "@eslint/js": "^9.13.0",
     "@iconify-icon/react": "^2.1.0",
-    "@types/fluent-ffmpeg": "^2.1.27",
+    "@types/better-sqlite3": "^7.6.12",
     "@types/gulp": "^4.0.17",
     "@types/react": "^18.3.12",
     "@types/react-dom": "^18.3.1",
+    "@types/screenshot-desktop": "^1.12.3",
+    "@types/sqlstring": "^2.3.2",
     "@vitejs/plugin-react": "^4.3.3",
     "autoprefixer": "^10.4.19",
     "concurrently": "^9.0.1",
@@ -62,6 +66,7 @@
     "gulp-clean": "^0.4.0",
     "gulp-typescript": "6.0.0-alpha.1",
     "postcss": "^8.4.38",
+    "screenshot-desktop": "^1.15.0",
     "tailwindcss": "^3.4.15",
     "typescript": "~5.6.2",
     "typescript-eslint": "^8.11.0",

src/electron/backend/init.ts

@@ -0,0 +1,90 @@
+import * as path from "path";
+import Database from "better-sqlite3";
+import { __dirname } from "../dirname.js";
+import { getDatabasePath } from "../utils/backend.js";
+
+function getLibSimpleExtensionPath(): string {
+	return path.join(__dirname, "bin", process.platform, "libsimple/libsimple.dylib");
+}
+
+export function initDatabase() {
+	const dbPath = getDatabasePath();
+	const db = new Database(dbPath, { verbose: console.log });
+	const libSimpleExtensionPath = getLibSimpleExtensionPath();
+
+	db.loadExtension(libSimpleExtensionPath);
+
+	db.exec(`
+        CREATE TABLE IF NOT EXISTS frame (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            createAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+            imgFilename TEXT,
+            segmentID INTEGER NULL,
+            videoPath TEXT NULL,
+            videoFrameIndex INTEGER NULL,
+            collectionID INTEGER NULL,
+            encoded BOOLEAN DEFAULT 0,
+            FOREIGN KEY (segmentID) REFERENCES segements (id)
+        );
+    `);
+
+	db.exec(`
+        CREATE TABLE IF NOT EXISTS recognition_data (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            frameID INTEGER,
+            data TEXT,
+            text TEXT,
+            FOREIGN KEY (frameID) REFERENCES frame (id)
+        );
+    `);
+
+	db.exec(`
+        CREATE TABLE IF NOT EXISTS segements(
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            startAt TIMESTAMP,
+            endAt TIMESTAMP,
+            title TEXT,
+            appName TEXT,
+            appPath TEXT,
+            text TEXT,
+            type TEXT,
+            appBundleID TEXT NULL,
+            url TEXT NULL
+        );
+    `);
+
+	db.exec(`
+        CREATE VIRTUAL TABLE IF NOT EXISTS text_search USING fts5(
+            id UNINDEXED,
+            frameID UNINDEXED,
+            data,
+            text
+        );
+    `);
+
+	db.exec(`
+        CREATE TRIGGER IF NOT EXISTS recognition_data_after_insert AFTER INSERT ON recognition_data
+        BEGIN
+            INSERT INTO text_search (id, frameID, data, text)
+            VALUES (NEW.id, NEW.frameID, NEW.data, NEW.text);
+        END;
+    `);
+
+	db.exec(`
+        CREATE TRIGGER IF NOT EXISTS recognition_data_after_update AFTER UPDATE ON recognition_data
+        BEGIN
+            UPDATE text_search
+            SET frameID = NEW.frameID, data = NEW.data, text = NEW.text
+            WHERE id = NEW.id;
+        END;
+    `);
+
+	db.exec(`
+        CREATE TRIGGER IF NOT EXISTS recognition_data_after_delete AFTER DELETE ON recognition_data
+        BEGIN
+            DELETE FROM text_search WHERE id = OLD.id;
+        END;
+    `);
+
+	return db;
+}

src/electron/backend/screenshot.ts

@@ -0,0 +1,23 @@
+import screenshot from "screenshot-desktop";
+import { getScreenshotsPath } from "../utils/backend.js";
+import { join } from "path";
+import { Database }from "better-sqlite3";
+import SqlString from "sqlstring";
+
+export function startScreenshotLoop(db: Database) {
+	return setInterval(() => {
+		const timestamp = new Date().getTime();
+		const screenshotPath = getScreenshotsPath();
+		const filename = join(screenshotPath, `${timestamp}.png`);
+		screenshot({filename: filename}).then((absolutePath) => {
+			const SQL = SqlString.format(
+				"INSERT INTO frame (imgFilename) VALUES (?)",
+				[absolutePath]
+			);
+			db.exec(SQL);
+		}).catch((err) => {
+			console.error(err);
+		});
+	}, 2000);
+}
+

src/electron/createWindow.ts

@@ -1,6 +1,6 @@
 import { app, BrowserWindow, screen } from "electron";
 import { join } from "path";
-import { __dirname } from "./utils.js";
+import { __dirname } from "./dirname.js";
 import windowStateManager from "electron-window-state";
 
 function loadURL(window: BrowserWindow, path = "", vitePort: string) {
@@ -34,11 +34,11 @@ export function createSettingsWindow(vitePort: string, closeCallBack: Function)
 		},
 		titleBarStyle: "hiddenInset",
 		resizable: false,
+		show: false,
 	});
 	windowState.manage(window);
-	window.once("ready-to-show", () => {
+	window.on("show", () => {
-		window.show();
+		app.dock.show();
-		window.focus();
 	});
 	window.on("close", (e) => {
 		window.hide();
@@ -46,7 +46,8 @@ export function createSettingsWindow(vitePort: string, closeCallBack: Function)
 		e.preventDefault();
 	});
 	window.once("close", () => {
-		window.hide()
+		window.hide();
+		app.dock.hide();
 	});
 	loadURL(window, "settings", vitePort);
 	return window;
@@ -77,17 +78,11 @@ export function createMainWindow(vitePort: string, closeCallBack: Function) {
 		},
 		roundedCorners: false,
 		transparent: true,
+		show: false
 	});
 
 	windowState.manage(window);
 
-	window.once("ready-to-show", () => {
-		window.show();
-		window.setAlwaysOnTop(true, "screen-saver");
-		window.setBounds({ x: 0, y: 0, width, height });
-		window.focus();
-	});
-
 	window.on("close", () => {
 		windowState.saveState(window);
 	});

src/electron/dirname.ts

@@ -0,0 +1,4 @@
+import path from "path";
+import { fileURLToPath } from "url";
+
+export const __dirname = path.dirname(fileURLToPath(import.meta.url));

src/electron/i18n.ts

@@ -2,7 +2,7 @@ import { join } from "path";
 import i18n from "i18next";
 import fs from "fs";
 import { app } from "electron";
-import { __dirname } from "./utils.js";
+import { __dirname } from "./dirname.js";
 
 
 /**

src/electron/index.ts

@@ -1,10 +1,12 @@
-import { app, BrowserWindow, globalShortcut, Menu, nativeImage, Tray } from "electron";
+import { app, BrowserWindow, globalShortcut, Menu, nativeImage, screen, Tray } from "electron";
 import contextMenu from "electron-context-menu";
 import { join } from "path";
 import initI18n from "./i18n.js";
 import { createMainWindow, createSettingsWindow } from "./createWindow.js";
-import { __dirname, captureScreen, getFirstCaptureScreenDeviceId } from "./utils.js";
+import { initDatabase } from "./backend/init.js";
-import * as fs from "fs";
+import { Database } from "better-sqlite3";
+import { startScreenshotLoop } from "./backend/screenshot.js";
+import { __dirname } from "./dirname.js";
 
 const i18n = initI18n();
 
@@ -12,19 +14,12 @@ const t = i18n.t.bind(i18n);
 const port = process.env.PORT || "5173";
 const dev = !app.isPackaged;
 
-let tray = null;
+let tray: null | Tray = null;
+let dbConnection: null | Database = null;
+let screenshotInterval: null | NodeJS.Timeout = null;
 
-async function c() {
+let mainWindow: BrowserWindow | null;
-	const screenshotpath = join(__dirname, "screenshot.png");
+let settingsWindow: BrowserWindow | null;
-	const ffmpegPath = join(__dirname, dev ? "bin/macos/ffmpeg" : "../../bin/macos/ffmpeg");
-	const deviceID = await getFirstCaptureScreenDeviceId(ffmpegPath);
-	if (deviceID) {
-		await captureScreen(ffmpegPath, deviceID, screenshotpath);
-		const screenshotData = fs.readFileSync(screenshotpath, "base64");
-		return screenshotData;
-	}
-	return null;
-}
 
 function createTray() {
 	const pathRoot: string = dev ? "./src/electron/assets/" : join(__dirname, "./assets/");
@@ -36,26 +31,18 @@ function createTray() {
 	const contextMenu = Menu.buildFromTemplate([
 		{
 			label: t("tray.showMainWindow"),
-			click: async () => {
+			click: () => {
-				if (!mainWindow) mainWindow = createMainWindow(port, () => (mainWindow = null));
+				const display = screen.getPrimaryDisplay();
-				mainWindow!.webContents.send("fromMain", null);
+				const { width, height } = display.bounds;
-				mainWindow!.setIgnoreMouseEvents(true);
 				mainWindow!.show();
-				c()
+				mainWindow!.setAlwaysOnTop(true, "screen-saver");
-					.then((data) => {
+				mainWindow!.setBounds({ x: 0, y: 0, width, height });
-						mainWindow!.webContents.send("fromMain", data);
+				mainWindow!.focus();
-						mainWindow!.setIgnoreMouseEvents(false);
-					})
-					.catch((err) => {
-						console.error(err);
-					});
 			}
 		},
 		{
 			label: t("tray.showSettingsWindow"),
 			click: () => {
-				if (!settingsWindow)
-					settingsWindow = createSettingsWindow(port, () => (settingsWindow = null));
 				settingsWindow!.show();
 			}
 		},
@@ -72,9 +59,6 @@ function createTray() {
 	tray.setToolTip("OpenRewind");
 }
 
-let mainWindow: BrowserWindow | null;
-let settingsWindow: BrowserWindow | null;
-
 contextMenu({
 	showLookUpSelection: true,
 	showSearchWithGoogle: true,
@@ -88,8 +72,12 @@ app.on("activate", () => {});
 
 app.on("ready", () => {
 	createTray();
+	dbConnection = initDatabase();
+	screenshotInterval = startScreenshotLoop(dbConnection);
+	mainWindow = createMainWindow(port, () => (mainWindow = null));
+	settingsWindow = createSettingsWindow(port, () => (settingsWindow = null));
 	globalShortcut.register("Escape", () => {
-		if (!mainWindow) return;
+		if (!mainWindow || !mainWindow.isVisible()) return;
 		mainWindow.hide();
 	});
 });

src/electron/utils.ts

@@ -1,35 +0,0 @@
-import path from "path";
-import { fileURLToPath } from "url";
-import { exec } from "child_process";
-
-export const __dirname = path.dirname(fileURLToPath(import.meta.url));
-
-export function getFirstCaptureScreenDeviceId(ffmpegPath: string): Promise<string | null> {
-	return new Promise((resolve, reject) => {
-		exec(`${ffmpegPath} -f avfoundation -list_devices true -i ""`, (error, stdout, stderr) => {
-			// stderr contains the output we need to parse
-			const output = stderr;
-			const captureScreenRegex = /\[(\d+)]\s+Capture screen \d+/g;
-			const match = captureScreenRegex.exec(output);
-
-			if (match) {
-				resolve(match[1]);
-			} else {
-				resolve(null);
-			}
-		});
-	});
-}
-
-export function captureScreen(ffmpegPath: string ,deviceId: string, outputPath: string): Promise<void> {
-	return new Promise((resolve, reject) => {
-		exec(`${ffmpegPath} -f avfoundation -pixel_format uyvy422 -i ${deviceId} -y -frames:v 1 ${outputPath}`,
-			(error, _stdout, _stderr) => {
-				if (error) {
-					reject(error);
-				} else {
-					resolve();
-				}
-			});
-	});
-}

src/electron/utils/backend.ts

@@ -0,0 +1,51 @@
+import path from "path";
+import os from "os";
+import fs from "fs";
+
+export function getUserDataDir() {
+	switch (process.platform) {
+		case "win32":
+			return path.join(process.env.APPDATA!, "OpenRewind", "Record Data");
+		case "darwin":
+			return path.join(os.homedir(), "Library", "Application Support", "OpenRewind", "Record Data");
+		case "linux":
+			return path.join(os.homedir(), ".config", "OpenRewind", "Record Data");
+		default:
+			throw new Error("Unsupported platform");
+	}
+}
+
+export function createDataDir() {
+	const dataDir = getUserDataDir();
+	if (!fs.existsSync(dataDir)) {
+		fs.mkdirSync(dataDir, { recursive: true });
+	}
+	return dataDir;
+}
+
+export function createTempDir() {
+	const tempDir = path.join(getUserDataDir(), "temp");
+	if (!fs.existsSync(tempDir)) {
+		fs.mkdirSync(tempDir, { recursive: true });
+	}
+	return tempDir;
+}
+
+export function getDatabasePath() {
+	const dataDir = createDataDir();
+	return path.join(dataDir, "main.db");
+}
+
+export function getScreenshotsPath() {
+	const tempDir = createTempDir();
+	const screenshotsDir = path.join(tempDir, "screenshots");
+	if (!fs.existsSync(screenshotsDir)) {
+		fs.mkdirSync(screenshotsDir, { recursive: true });
+	}
+	return screenshotsDir;
+}
+
+export function getRecordingsPath() {
+	const dataDir = createDataDir();
+	return path.join(dataDir, "recordings");
+}