add: tokenID added to JWT in endpoint GET /captcha/:id/result

packages/backend/routes/captcha/[id]/result/GET.ts

@@ -1,11 +1,14 @@
 import { Context } from "hono";
-import { Bindings, BlankEnv, BlankInput } from "hono/types";
+import { Bindings, BlankEnv } from "hono/types";
 import { ErrorResponse } from "src/schema";
 import { createHandlers } from "src/utils.ts";
 import * as jose from "jose";
+import { generateRandomId } from "@core/lib/randomID.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 
 interface CaptchaResponse {
 	success: boolean;
+	difficulty?: number;
 	error?: string;
 }
 
@@ -13,7 +16,7 @@ const getChallengeVerificationResult = async (id: string, ans: string) => {
 	const baseURL = process.env["UCAPTCHA_URL"];
 	const url = new URL(baseURL);
 	url.pathname = `/challenge/${id}/validation`;
-	const res = await fetch(url.toString(), {
+	return await fetch(url.toString(), {
 		method: "POST",
 		headers: {
 			"Content-Type": "application/json"
@@ -22,11 +25,10 @@ const getChallengeVerificationResult = async (id: string, ans: string) => {
 			y: ans
 		})
 	});
-	return res;
 };
 
 export const verifyChallengeHandler = createHandlers(
-	async (c: Context<BlankEnv & { Bindings: Bindings }, "/captcha/:id/result", BlankInput>) => {
+	async (c: Context<BlankEnv & { Bindings: Bindings }, "/captcha/:id/result">) => {
 		const id = c.req.param("id");
 		const ans = c.req.query("ans");
 		if (!ans) {
@@ -76,7 +78,11 @@ export const verifyChallengeHandler = createHandlers(
 		const jwtSecret = new TextEncoder().encode(secret);
 		const alg = "HS256";
 
-		const jwt = await new jose.SignJWT()
+
+		const tokenID = generateRandomId(10);
+		const EXPIRE_FIVE_MINUTES = 300;
+		await lockManager.acquireLock(tokenID, EXPIRE_FIVE_MINUTES);
+		const jwt = await new jose.SignJWT({ difficulty: data.difficulty!, id: tokenID })
 			.setProtectedHeader({ alg })
 			.setIssuedAt()
 			.sign(jwtSecret);

packages/core/deno.json

@@ -1,17 +0,0 @@
-{
-    "name": "@cvsa/core",
-    "exports": "./main.ts",
-    "imports": {
-        "ioredis": "npm:ioredis",
-        "log/": "./log/",
-        "db/": "./db/",
-        "$std/": "https://deno.land/std@0.216.0/",
-        "mq/": "./mq/",
-        "chalk": "npm:chalk",
-        "winston": "npm:winston",
-        "logform": "npm:logform",
-        "@core/": "./",
-        "child_process": "node:child_process",
-        "util": "node:util"
-    }
-}

packages/core/lib/randomID.ts

@@ -0,0 +1,142 @@
+const getSecureRandomInt = (max: number): number => {
+	const array = new Uint32Array(1);
+	crypto.getRandomValues(array);
+	// Using modulo bias is technically present, but negligible here because the space (56) is tiny compared to 2^32.
+	return array[0] % max;
+}
+
+
+/**
+ * Generates a random ID with characteristics similar to UUIDv7,
+ * incorporating a timestamp prefix for sortability and a random suffix,
+ * using a customizable length and a specific character set.
+ *
+ * This function aims for sortability by placing a time-based component at the beginning,
+ * similar to UUIDv7, while allowing a variable total length and using a character set
+ * designed to avoid visually similar characters.
+ *
+ * The character set includes uppercase and lowercase letters and numbers,
+ * excluding visually similar characters (0, O, I, l, 1).
+ *
+ * **Length Reference**:
+ *
+ * With a collision probability of **0.1%**,
+ * the maximum ID generation rate per millisecond for the following lengths is:
+ * - **10**: 1.8 IDs / ms or 1,844 QPS
+ * - **12**: 27 IDs / ms or 26,998 QPS
+ * - **16**: 5784 IDs / ms or 5,784,295 QPS
+ *
+ * With a collision probability of **0.001%**,
+ * the maximum ID generation rate per millisecond for the following lengths is:
+ * - **11**: 1.5 IDs / ms or 1,520 QPS
+ * - **14**: 85 IDs / ms or 85,124 QPS
+ * - **16**: 1246 IDs / ms or 1,245,983 QPS
+ *
+ * With a collision probability of **0.00001%**,
+ * the maximum ID generation rate per millisecond for the following lengths is:
+ * - **14**: 18 IDs / ms or 18,339 QPS
+ * - **15**: 70 IDs / ms or 70,164 QPS
+ * - **16**: 1246 IDs / ms or 268,438 QPS
+ *
+ * The formula: max_qps = 1000 * (2 * (56**(length - 8)) * -log(1 - prob))**(1/3)
+ *
+ * @param length The desired total length of the ID. Must be at least 8.
+ * @returns A sortable random ID string of the specified length.
+ * @throws Error if the requested length is less than the minimum required for the timestamp prefix (8).
+ */
+export function generateRandomId(length: number): string {
+	// Character set excluding 0, O, I, l, 1
+	const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters
+	const base = allowedChars.length; // 56
+	const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part to ensure sortability
+
+	if (length < TIMESTAMP_PREFIX_LENGTH) {
+		throw new Error(`Length must be at least ${TIMESTAMP_PREFIX_LENGTH} to include the timestamp prefix.`);
+	}
+
+	// --- Generate Timestamp Prefix ---
+	const timestamp = Date.now(); // Milliseconds since epoch (Unix time)
+	let timestampBaseString = "";
+	let tempTimestamp = timestamp;
+	const firstChar = allowedChars[0]; // Character for padding ('a')
+
+	// Convert timestamp to a base-56 string
+	// We process from the least significant "digit" to the most significant
+	while (tempTimestamp > 0) {
+		timestampBaseString = allowedChars[tempTimestamp % base] + timestampBaseString;
+		tempTimestamp = Math.floor(tempTimestamp / base);
+	}
+
+	// Pad the timestamp string at the beginning to ensure a fixed length.
+	// This is crucial for chronological sortability of the generated IDs.
+	while (timestampBaseString.length < TIMESTAMP_PREFIX_LENGTH) {
+		timestampBaseString = firstChar + timestampBaseString;
+	}
+
+	// Although highly unlikely with an 8-character prefix using base 56 for current timestamps,
+	// this would truncate if the timestamp string somehow exceeded the prefix length.
+	if (timestampBaseString.length > TIMESTAMP_PREFIX_LENGTH) {
+		timestampBaseString = timestampBaseString.substring(timestampBaseString.length - TIMESTAMP_PREFIX_LENGTH);
+	}
+
+	// --- Generate Random Suffix ---
+	const randomLength = length - TIMESTAMP_PREFIX_LENGTH;
+	let randomSuffix = "";
+	const allowedCharsLength = allowedChars.length;
+
+	for (let i = 0; i < randomLength; i++) {
+		const randomIndex = getSecureRandomInt(allowedCharsLength);
+		randomSuffix += allowedChars[randomIndex];
+	}
+
+	// --- Concatenate and Return ---
+	return timestampBaseString + randomSuffix;
+}
+
+/**
+ * Decodes the timestamp (in milliseconds since epoch) from an ID
+ * generated by a function that uses a fixed-length timestamp prefix
+ * encoded in a specific base and character set (like the previous example).
+ *
+ * It extracts the timestamp prefix and converts it back from the
+ * custom base-56 encoding to a number.
+ *
+ * @param id The ID string containing the timestamp prefix.
+ * @returns The timestamp in milliseconds since epoch.
+ * @throws Error if the ID is too short or contains invalid characters in the timestamp prefix.
+ */
+export function decodeTimestampFromId(id: string): number {
+	// Character set must match the encoding function used to generate the ID
+	const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters
+	const base = allowedChars.length; // 56
+	const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part
+
+	if (id.length < TIMESTAMP_PREFIX_LENGTH) {
+		throw new Error(
+			`ID must be at least ${TIMESTAMP_PREFIX_LENGTH} characters long to contain a timestamp prefix.`
+		);
+	}
+
+	// Extract the timestamp prefix from the beginning of the ID string
+	const timestampPrefix = id.substring(0, TIMESTAMP_PREFIX_LENGTH);
+	let timestamp = 0;
+
+	// Convert the base-56 timestamp string back to a number
+	// Iterate through the prefix characters from left to right (most significant 'digit')
+	for (let i = 0; i < timestampPrefix.length; i++) {
+		const char = timestampPrefix[i];
+		// Find the index (value) of the character in the allowed character set
+		const charIndex = allowedChars.indexOf(char);
+
+		if (charIndex === -1) {
+			// If a character is not in the allowed set, the ID is likely invalid
+			throw new Error(`Invalid character "${char}" found in timestamp prefix.`);
+		}
+
+		// Standard base conversion: accumulate the value
+		// For each digit, multiply the current total by the base and add the digit's value
+		timestamp = timestamp * base + charIndex;
+	}
+
+	return timestamp;
+}

packages/core/mq/lockManager.ts

@@ -1,5 +1,5 @@
 import { Redis } from "ioredis";
-import { redis } from "../../core/db/redis.ts";
+import { redis } from "@core/db/redis.ts";
 
 class LockManager {
 	private redis: Redis;

packages/core/net/delegate.ts

@@ -1,8 +1,7 @@
 import logger from "@core/log/logger.ts";
 import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts";
-import { redis } from "db/redis.ts";
 import { ReplyError } from "ioredis";
-import { SECOND } from "../const/time.ts";
+import { SECOND } from "@core/const/time.ts";
 import { spawn, SpawnOptions } from "child_process";
 
 export function spawnPromise(

packages/core/package.json

@@ -1,13 +1,16 @@
 {
-  "name": "core",
-  "dependencies": {
-    "chalk": "^5.4.1",
-    "ioredis": "^5.6.1",
-    "logform": "^2.7.0",
-    "postgres": "^3.4.5",
-    "winston": "^3.17.0"
-  },
-  "devDependencies": {
-    "@types/ioredis": "^5.0.0"
-  }
-}
+	"name": "core",
+	"scripts": {
+		"test": "bun --env-file=.env.test run vitest"
+	},
+	"dependencies": {
+		"chalk": "^5.4.1",
+		"ioredis": "^5.6.1",
+		"logform": "^2.7.0",
+		"postgres": "^3.4.5",
+		"winston": "^3.17.0"
+	},
+	"devDependencies": {
+		"@types/ioredis": "^5.0.0"
+	}
+}

packages/core/test/lib/randomID.test.ts

@@ -0,0 +1,80 @@
+import { describe, expect, it } from "vitest";
+import { generateRandomId, decodeTimestampFromId } from "@core/lib/randomID.ts";
+
+describe("generateRandomId", () => {
+	it("should throw an error if the requested length is less than 8", () => {
+		expect(() => generateRandomId(7)).toThrowError("Length must be at least 8 to include the timestamp prefix.");
+	});
+
+	it("should generate an ID of the specified length", () => {
+		const length = 15;
+		const id = generateRandomId(length);
+		expect(id).toHaveLength(length);
+	});
+
+	it("should generate an ID with a timestamp prefix of length 8", () => {
+		const id = generateRandomId(12);
+		expect(id).toHaveProperty("substring");
+		expect(id).toHaveProperty("length");
+		expect(id.length).toBeGreaterThanOrEqual(8);
+	});
+
+	it("should generate an ID containing only allowed characters", () => {
+		const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789";
+		const id = generateRandomId(20);
+		for (const char of id) {
+			expect(allowedChars).toContain(char);
+		}
+	});
+
+	it("should generate IDs that are sortable by creation time", () => {
+		const id1 = generateRandomId(10);
+		// Simulate a slight delay to ensure different timestamps
+		return new Promise((resolve) => {
+			setTimeout(() => {
+				const id2 = generateRandomId(10);
+				expect(id2 >= id1).toBe(true);
+				resolve(null);
+			}, 10);
+		});
+	});
+});
+
+describe("decodeTimestampFromId", () => {
+	it("should throw an error if the ID length is less than 8", () => {
+		expect(() => decodeTimestampFromId("abcdefg")).toThrowError(
+			"ID must be at least 8 characters long to contain a timestamp prefix."
+		);
+	});
+
+	it("should throw an error if the timestamp prefix contains invalid characters", () => {
+		const invalidId = "0bcdefghijk";
+		expect(() => decodeTimestampFromId(invalidId)).toThrowError('Invalid character "0" found in timestamp prefix.');
+	});
+
+	it("should correctly decode the timestamp from a generated ID", () => {
+		const now = Date.now();
+		// Mock Date.now to control the timestamp for testing
+		const originalDateNow = Date.now;
+		global.Date.now = () => now;
+		const id = generateRandomId(16);
+		global.Date.now = originalDateNow; // Restore original Date.now
+
+		const decodedTimestamp = decodeTimestampFromId(id);
+		// Allow a small margin for potential timing differences in test execution
+		expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1);
+		expect(decodedTimestamp).toBeLessThanOrEqual(now + 1);
+	});
+
+	it("should correctly decode the timestamp even with a longer ID", () => {
+		const now = Date.now();
+		const originalDateNow = Date.now;
+		global.Date.now = () => now;
+		const id = generateRandomId(20);
+		global.Date.now = originalDateNow;
+
+		const decodedTimestamp = decodeTimestampFromId(id);
+		expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1);
+		expect(decodedTimestamp).toBeLessThanOrEqual(now + 1);
+	});
+});

packages/crawler/mq/exec/archiveSnapshots.ts

@@ -1,7 +1,7 @@
 import { Job } from "bullmq";
 import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { getLatestVideoSnapshot } from "db/snapshot.ts";
 import { HOUR, MINUTE } from "@core/const/time.ts";
 import { sql } from "@core/db/dbNew";

packages/crawler/mq/exec/classifyVideo.ts

@@ -3,7 +3,7 @@ import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "
 import Akari from "ml/akari.ts";
 import { ClassifyVideoQueue } from "mq/index.ts";
 import logger from "@core/log/logger.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { aidExistsInSongs } from "db/songs.ts";
 import { insertIntoSongs } from "mq/task/collectSongs.ts";
 import { scheduleSnapshot } from "db/snapshotSchedule.ts";

packages/crawler/mq/exec/dispatchRegularSnapshots.ts

@@ -4,7 +4,7 @@ import { truncate } from "utils/truncate.ts";
 import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { HOUR, MINUTE, WEEK } from "@core/const/time.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts";
 import { sql } from "@core/db/dbNew.ts";
 

packages/crawler/mq/exec/snapshotVideo.ts

@@ -2,7 +2,7 @@ import { Job } from "bullmq";
 import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
 import { insertVideoSnapshot } from "mq/task/getVideoStats.ts";
 import { getSongsPublihsedAt } from "db/songs.ts";

packages/crawler/src/filterWorker.ts

@@ -3,7 +3,7 @@ import { redis } from "@core/db/redis.ts";
 import logger from "@core/log/logger.ts";
 import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts";
 import { WorkerError } from "mq/schema.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import Akari from "ml/akari.ts";
 
 const shutdown = async (signal: string) => {

packages/crawler/src/worker.ts

@@ -14,7 +14,7 @@ import {
 } from "mq/exec/executors.ts";
 import { redis } from "@core/db/redis.ts";
 import logger from "@core/log/logger.ts";
-import { lockManager } from "mq/lockManager.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { WorkerError } from "mq/schema.ts";
 
 const releaseLockForJob = async (name: string) => {