diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts index 6a507da..cffab6d 100644 --- a/packages/backend/routes/captcha/[id]/result/GET.ts +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -1,11 +1,14 @@ import { Context } from "hono"; -import { Bindings, BlankEnv, BlankInput } from "hono/types"; +import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; import { createHandlers } from "src/utils.ts"; import * as jose from "jose"; +import { generateRandomId } from "@core/lib/randomID.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; interface CaptchaResponse { success: boolean; + difficulty?: number; error?: string; } @@ -13,7 +16,7 @@ const getChallengeVerificationResult = async (id: string, ans: string) => { const baseURL = process.env["UCAPTCHA_URL"]; const url = new URL(baseURL); url.pathname = `/challenge/${id}/validation`; - const res = await fetch(url.toString(), { + return await fetch(url.toString(), { method: "POST", headers: { "Content-Type": "application/json" @@ -22,11 +25,10 @@ const getChallengeVerificationResult = async (id: string, ans: string) => { y: ans }) }); - return res; }; export const verifyChallengeHandler = createHandlers( - async (c: Context) => { + async (c: Context) => { const id = c.req.param("id"); const ans = c.req.query("ans"); if (!ans) { @@ -76,7 +78,11 @@ export const verifyChallengeHandler = createHandlers( const jwtSecret = new TextEncoder().encode(secret); const alg = "HS256"; - const jwt = await new jose.SignJWT() + + const tokenID = generateRandomId(10); + const EXPIRE_FIVE_MINUTES = 300; + await lockManager.acquireLock(tokenID, EXPIRE_FIVE_MINUTES); + const jwt = await new jose.SignJWT({ difficulty: data.difficulty!, id: tokenID }) .setProtectedHeader({ alg }) .setIssuedAt() .sign(jwtSecret); diff --git a/packages/core/deno.json b/packages/core/deno.json deleted file mode 100644 index b1027e9..0000000 --- a/packages/core/deno.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@cvsa/core", - "exports": "./main.ts", - "imports": { - "ioredis": "npm:ioredis", - "log/": "./log/", - "db/": "./db/", - "$std/": "https://deno.land/std@0.216.0/", - "mq/": "./mq/", - "chalk": "npm:chalk", - "winston": "npm:winston", - "logform": "npm:logform", - "@core/": "./", - "child_process": "node:child_process", - "util": "node:util" - } -} diff --git a/packages/core/lib/randomID.ts b/packages/core/lib/randomID.ts new file mode 100644 index 0000000..477fe4a --- /dev/null +++ b/packages/core/lib/randomID.ts @@ -0,0 +1,142 @@ +const getSecureRandomInt = (max: number): number => { + const array = new Uint32Array(1); + crypto.getRandomValues(array); + // Using modulo bias is technically present, but negligible here because the space (56) is tiny compared to 2^32. + return array[0] % max; +} + + +/** + * Generates a random ID with characteristics similar to UUIDv7, + * incorporating a timestamp prefix for sortability and a random suffix, + * using a customizable length and a specific character set. + * + * This function aims for sortability by placing a time-based component at the beginning, + * similar to UUIDv7, while allowing a variable total length and using a character set + * designed to avoid visually similar characters. + * + * The character set includes uppercase and lowercase letters and numbers, + * excluding visually similar characters (0, O, I, l, 1). + * + * **Length Reference**: + * + * With a collision probability of **0.1%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **10**: 1.8 IDs / ms or 1,844 QPS + * - **12**: 27 IDs / ms or 26,998 QPS + * - **16**: 5784 IDs / ms or 5,784,295 QPS + * + * With a collision probability of **0.001%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **11**: 1.5 IDs / ms or 1,520 QPS + * - **14**: 85 IDs / ms or 85,124 QPS + * - **16**: 1246 IDs / ms or 1,245,983 QPS + * + * With a collision probability of **0.00001%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **14**: 18 IDs / ms or 18,339 QPS + * - **15**: 70 IDs / ms or 70,164 QPS + * - **16**: 1246 IDs / ms or 268,438 QPS + * + * The formula: max_qps = 1000 * (2 * (56**(length - 8)) * -log(1 - prob))**(1/3) + * + * @param length The desired total length of the ID. Must be at least 8. + * @returns A sortable random ID string of the specified length. + * @throws Error if the requested length is less than the minimum required for the timestamp prefix (8). + */ +export function generateRandomId(length: number): string { + // Character set excluding 0, O, I, l, 1 + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters + const base = allowedChars.length; // 56 + const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part to ensure sortability + + if (length < TIMESTAMP_PREFIX_LENGTH) { + throw new Error(`Length must be at least ${TIMESTAMP_PREFIX_LENGTH} to include the timestamp prefix.`); + } + + // --- Generate Timestamp Prefix --- + const timestamp = Date.now(); // Milliseconds since epoch (Unix time) + let timestampBaseString = ""; + let tempTimestamp = timestamp; + const firstChar = allowedChars[0]; // Character for padding ('a') + + // Convert timestamp to a base-56 string + // We process from the least significant "digit" to the most significant + while (tempTimestamp > 0) { + timestampBaseString = allowedChars[tempTimestamp % base] + timestampBaseString; + tempTimestamp = Math.floor(tempTimestamp / base); + } + + // Pad the timestamp string at the beginning to ensure a fixed length. + // This is crucial for chronological sortability of the generated IDs. + while (timestampBaseString.length < TIMESTAMP_PREFIX_LENGTH) { + timestampBaseString = firstChar + timestampBaseString; + } + + // Although highly unlikely with an 8-character prefix using base 56 for current timestamps, + // this would truncate if the timestamp string somehow exceeded the prefix length. + if (timestampBaseString.length > TIMESTAMP_PREFIX_LENGTH) { + timestampBaseString = timestampBaseString.substring(timestampBaseString.length - TIMESTAMP_PREFIX_LENGTH); + } + + // --- Generate Random Suffix --- + const randomLength = length - TIMESTAMP_PREFIX_LENGTH; + let randomSuffix = ""; + const allowedCharsLength = allowedChars.length; + + for (let i = 0; i < randomLength; i++) { + const randomIndex = getSecureRandomInt(allowedCharsLength); + randomSuffix += allowedChars[randomIndex]; + } + + // --- Concatenate and Return --- + return timestampBaseString + randomSuffix; +} + +/** + * Decodes the timestamp (in milliseconds since epoch) from an ID + * generated by a function that uses a fixed-length timestamp prefix + * encoded in a specific base and character set (like the previous example). + * + * It extracts the timestamp prefix and converts it back from the + * custom base-56 encoding to a number. + * + * @param id The ID string containing the timestamp prefix. + * @returns The timestamp in milliseconds since epoch. + * @throws Error if the ID is too short or contains invalid characters in the timestamp prefix. + */ +export function decodeTimestampFromId(id: string): number { + // Character set must match the encoding function used to generate the ID + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters + const base = allowedChars.length; // 56 + const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part + + if (id.length < TIMESTAMP_PREFIX_LENGTH) { + throw new Error( + `ID must be at least ${TIMESTAMP_PREFIX_LENGTH} characters long to contain a timestamp prefix.` + ); + } + + // Extract the timestamp prefix from the beginning of the ID string + const timestampPrefix = id.substring(0, TIMESTAMP_PREFIX_LENGTH); + let timestamp = 0; + + // Convert the base-56 timestamp string back to a number + // Iterate through the prefix characters from left to right (most significant 'digit') + for (let i = 0; i < timestampPrefix.length; i++) { + const char = timestampPrefix[i]; + // Find the index (value) of the character in the allowed character set + const charIndex = allowedChars.indexOf(char); + + if (charIndex === -1) { + // If a character is not in the allowed set, the ID is likely invalid + throw new Error(`Invalid character "${char}" found in timestamp prefix.`); + } + + // Standard base conversion: accumulate the value + // For each digit, multiply the current total by the base and add the digit's value + timestamp = timestamp * base + charIndex; + } + + return timestamp; +} diff --git a/packages/crawler/mq/lockManager.ts b/packages/core/mq/lockManager.ts similarity index 96% rename from packages/crawler/mq/lockManager.ts rename to packages/core/mq/lockManager.ts index fefae7b..129001a 100644 --- a/packages/crawler/mq/lockManager.ts +++ b/packages/core/mq/lockManager.ts @@ -1,5 +1,5 @@ import { Redis } from "ioredis"; -import { redis } from "../../core/db/redis.ts"; +import { redis } from "@core/db/redis.ts"; class LockManager { private redis: Redis; diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index e1518c0..6d7d6d7 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,8 +1,7 @@ import logger from "@core/log/logger.ts"; import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts"; -import { redis } from "db/redis.ts"; import { ReplyError } from "ioredis"; -import { SECOND } from "../const/time.ts"; +import { SECOND } from "@core/const/time.ts"; import { spawn, SpawnOptions } from "child_process"; export function spawnPromise( diff --git a/packages/core/package.json b/packages/core/package.json index a11ede0..9c291cb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,13 +1,16 @@ { - "name": "core", - "dependencies": { - "chalk": "^5.4.1", - "ioredis": "^5.6.1", - "logform": "^2.7.0", - "postgres": "^3.4.5", - "winston": "^3.17.0" - }, - "devDependencies": { - "@types/ioredis": "^5.0.0" - } -} \ No newline at end of file + "name": "core", + "scripts": { + "test": "bun --env-file=.env.test run vitest" + }, + "dependencies": { + "chalk": "^5.4.1", + "ioredis": "^5.6.1", + "logform": "^2.7.0", + "postgres": "^3.4.5", + "winston": "^3.17.0" + }, + "devDependencies": { + "@types/ioredis": "^5.0.0" + } +} diff --git a/packages/core/test/lib/randomID.test.ts b/packages/core/test/lib/randomID.test.ts new file mode 100644 index 0000000..71b859c --- /dev/null +++ b/packages/core/test/lib/randomID.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { generateRandomId, decodeTimestampFromId } from "@core/lib/randomID.ts"; + +describe("generateRandomId", () => { + it("should throw an error if the requested length is less than 8", () => { + expect(() => generateRandomId(7)).toThrowError("Length must be at least 8 to include the timestamp prefix."); + }); + + it("should generate an ID of the specified length", () => { + const length = 15; + const id = generateRandomId(length); + expect(id).toHaveLength(length); + }); + + it("should generate an ID with a timestamp prefix of length 8", () => { + const id = generateRandomId(12); + expect(id).toHaveProperty("substring"); + expect(id).toHaveProperty("length"); + expect(id.length).toBeGreaterThanOrEqual(8); + }); + + it("should generate an ID containing only allowed characters", () => { + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; + const id = generateRandomId(20); + for (const char of id) { + expect(allowedChars).toContain(char); + } + }); + + it("should generate IDs that are sortable by creation time", () => { + const id1 = generateRandomId(10); + // Simulate a slight delay to ensure different timestamps + return new Promise((resolve) => { + setTimeout(() => { + const id2 = generateRandomId(10); + expect(id2 >= id1).toBe(true); + resolve(null); + }, 10); + }); + }); +}); + +describe("decodeTimestampFromId", () => { + it("should throw an error if the ID length is less than 8", () => { + expect(() => decodeTimestampFromId("abcdefg")).toThrowError( + "ID must be at least 8 characters long to contain a timestamp prefix." + ); + }); + + it("should throw an error if the timestamp prefix contains invalid characters", () => { + const invalidId = "0bcdefghijk"; + expect(() => decodeTimestampFromId(invalidId)).toThrowError('Invalid character "0" found in timestamp prefix.'); + }); + + it("should correctly decode the timestamp from a generated ID", () => { + const now = Date.now(); + // Mock Date.now to control the timestamp for testing + const originalDateNow = Date.now; + global.Date.now = () => now; + const id = generateRandomId(16); + global.Date.now = originalDateNow; // Restore original Date.now + + const decodedTimestamp = decodeTimestampFromId(id); + // Allow a small margin for potential timing differences in test execution + expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); + expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); + }); + + it("should correctly decode the timestamp even with a longer ID", () => { + const now = Date.now(); + const originalDateNow = Date.now; + global.Date.now = () => now; + const id = generateRandomId(20); + global.Date.now = originalDateNow; + + const decodedTimestamp = decodeTimestampFromId(id); + expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); + expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); + }); +}); diff --git a/packages/crawler/mq/exec/archiveSnapshots.ts b/packages/crawler/mq/exec/archiveSnapshots.ts index 91411a9..c667416 100644 --- a/packages/crawler/mq/exec/archiveSnapshots.ts +++ b/packages/crawler/mq/exec/archiveSnapshots.ts @@ -1,7 +1,7 @@ import { Job } from "bullmq"; import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getLatestVideoSnapshot } from "db/snapshot.ts"; import { HOUR, MINUTE } from "@core/const/time.ts"; import { sql } from "@core/db/dbNew"; diff --git a/packages/crawler/mq/exec/classifyVideo.ts b/packages/crawler/mq/exec/classifyVideo.ts index 622eeed..6c3d37c 100644 --- a/packages/crawler/mq/exec/classifyVideo.ts +++ b/packages/crawler/mq/exec/classifyVideo.ts @@ -3,7 +3,7 @@ import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from " import Akari from "ml/akari.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { aidExistsInSongs } from "db/songs.ts"; import { insertIntoSongs } from "mq/task/collectSongs.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; diff --git a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts index 5c1652f..49a7893 100644 --- a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts @@ -4,7 +4,7 @@ import { truncate } from "utils/truncate.ts"; import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; import { HOUR, MINUTE, WEEK } from "@core/const/time.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts"; import { sql } from "@core/db/dbNew.ts"; diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts index 261b78b..59f05db 100644 --- a/packages/crawler/mq/exec/snapshotVideo.ts +++ b/packages/crawler/mq/exec/snapshotVideo.ts @@ -2,7 +2,7 @@ import { Job } from "bullmq"; import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts"; import { insertVideoSnapshot } from "mq/task/getVideoStats.ts"; import { getSongsPublihsedAt } from "db/songs.ts"; diff --git a/packages/crawler/src/filterWorker.ts b/packages/crawler/src/filterWorker.ts index 7ba2de6..ab85cba 100644 --- a/packages/crawler/src/filterWorker.ts +++ b/packages/crawler/src/filterWorker.ts @@ -3,7 +3,7 @@ import { redis } from "@core/db/redis.ts"; import logger from "@core/log/logger.ts"; import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts"; import { WorkerError } from "mq/schema.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import Akari from "ml/akari.ts"; const shutdown = async (signal: string) => { diff --git a/packages/crawler/src/worker.ts b/packages/crawler/src/worker.ts index ca644df..534f488 100644 --- a/packages/crawler/src/worker.ts +++ b/packages/crawler/src/worker.ts @@ -14,7 +14,7 @@ import { } from "mq/exec/executors.ts"; import { redis } from "@core/db/redis.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { WorkerError } from "mq/schema.ts"; const releaseLockForJob = async (name: string) => {