From 27728499336eaee25c193811d4979df1a0d60bcf Mon Sep 17 00:00:00 2001 From: alikia2x Date: Tue, 29 Apr 2025 04:52:58 +0800 Subject: [PATCH 01/12] update: .gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index c990962..1a69400 100644 --- a/.gitignore +++ b/.gitignore @@ -91,5 +91,4 @@ model/ *.db *.sqlite *.sqlite3 -db/ data/ \ No newline at end of file From 01f5e57864090a670d1c100a74331f67b81f2611 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sat, 10 May 2025 00:20:20 +0800 Subject: [PATCH 02/12] ref: dir structure of backend package --- packages/backend/db/config.ts | 2 +- packages/backend/db/db.ts | 2 +- .../db/{videoSnapshot.ts => snapshots.ts} | 0 .../{routes/root => lib/const}/singers.ts | 0 packages/backend/middleware/contentType.ts | 2 +- packages/backend/middleware/index.ts | 4 +- packages/backend/middleware/logger.ts | 2 +- .../backend/middleware/preetifyResponse.ts | 2 +- packages/backend/middleware/rateLimiters.ts | 2 +- packages/backend/routes/404.ts | 2 +- packages/backend/routes/index.ts | 44 ++++++++++++------- packages/backend/routes/ping.ts | 24 ---------- packages/backend/routes/ping/index.ts | 24 ++++++++++ packages/backend/routes/root/root.ts | 29 ------------ packages/backend/routes/user/index.ts | 1 + .../routes/{user.ts => user/register.ts} | 8 ++-- .../video/[id]/info.ts} | 24 +++++----- .../routes/{ => video/[id]}/snapshots.ts | 6 +-- packages/backend/routes/video/index.ts | 2 + packages/backend/src/main.ts | 2 +- packages/backend/src/routing.ts | 16 +++++++ packages/backend/src/schema.d.ts | 4 +- packages/backend/src/startServer.ts | 2 +- 23 files changed, 103 insertions(+), 101 deletions(-) rename packages/backend/db/{videoSnapshot.ts => snapshots.ts} (100%) rename packages/backend/{routes/root => lib/const}/singers.ts (100%) delete mode 100644 packages/backend/routes/ping.ts create mode 100644 packages/backend/routes/ping/index.ts delete mode 100644 packages/backend/routes/root/root.ts create mode 100644 packages/backend/routes/user/index.ts rename packages/backend/routes/{user.ts => user/register.ts} (99%) rename packages/backend/{db/videoInfo.ts => routes/video/[id]/info.ts} (84%) rename packages/backend/routes/{ => video/[id]}/snapshots.ts (94%) create mode 100644 packages/backend/routes/video/index.ts create mode 100644 packages/backend/src/routing.ts diff --git a/packages/backend/db/config.ts b/packages/backend/db/config.ts index 59ab5fa..9d0e514 100644 --- a/packages/backend/db/config.ts +++ b/packages/backend/db/config.ts @@ -35,7 +35,7 @@ export const postgresCredConfigNpm = { database: databaseNameCred, username: databaseUser, password: databasePassword -} +}; export const postgresConfigCred = { hostname: databaseHost, diff --git a/packages/backend/db/db.ts b/packages/backend/db/db.ts index 9643fa5..6dd488b 100644 --- a/packages/backend/db/db.ts +++ b/packages/backend/db/db.ts @@ -2,4 +2,4 @@ import postgres from "postgres"; import { postgresConfigNpm, postgresCredConfigNpm } from "./config"; export const sql = postgres(postgresConfigNpm); -export const sqlCred = postgres(postgresCredConfigNpm) \ No newline at end of file +export const sqlCred = postgres(postgresCredConfigNpm); diff --git a/packages/backend/db/videoSnapshot.ts b/packages/backend/db/snapshots.ts similarity index 100% rename from packages/backend/db/videoSnapshot.ts rename to packages/backend/db/snapshots.ts diff --git a/packages/backend/routes/root/singers.ts b/packages/backend/lib/const/singers.ts similarity index 100% rename from packages/backend/routes/root/singers.ts rename to packages/backend/lib/const/singers.ts diff --git a/packages/backend/middleware/contentType.ts b/packages/backend/middleware/contentType.ts index 7e47d75..edd4908 100644 --- a/packages/backend/middleware/contentType.ts +++ b/packages/backend/middleware/contentType.ts @@ -3,4 +3,4 @@ import { Context, Next } from "hono"; export const contentType = async (c: Context, next: Next) => { await next(); c.header("Content-Type", "application/json; charset=utf-8"); -}; \ No newline at end of file +}; diff --git a/packages/backend/middleware/index.ts b/packages/backend/middleware/index.ts index a43357e..4bd98af 100644 --- a/packages/backend/middleware/index.ts +++ b/packages/backend/middleware/index.ts @@ -8,7 +8,7 @@ import { logger } from "./logger.ts"; import { timing } from "hono/timing"; import { contentType } from "./contentType.ts"; -export function configureMiddleWares(app: Hono<{Variables: Variables }>) { +export function configureMiddleWares(app: Hono<{ Variables: Variables }>) { app.use("*", contentType); app.use(timing()); app.use("*", preetifyResponse); @@ -16,4 +16,4 @@ export function configureMiddleWares(app: Hono<{Variables: Variables }>) { app.post("/user", registerRateLimiter); app.all("/ping", bodyLimitForPing, ...pingHandler); -} \ No newline at end of file +} diff --git a/packages/backend/middleware/logger.ts b/packages/backend/middleware/logger.ts index 0625157..59a52b7 100644 --- a/packages/backend/middleware/logger.ts +++ b/packages/backend/middleware/logger.ts @@ -77,7 +77,7 @@ const defaultFormatter = (params) => { `${methodColor} ${params.method.padEnd(6)}${reset} ${params.path}` ); }; -type Ctx = Context +type Ctx = Context; export const logger = (config) => { const { formatter = defaultFormatter, output = console, skipPaths = [], skip = null } = config; diff --git a/packages/backend/middleware/preetifyResponse.ts b/packages/backend/middleware/preetifyResponse.ts index 5cf289b..ccc5944 100644 --- a/packages/backend/middleware/preetifyResponse.ts +++ b/packages/backend/middleware/preetifyResponse.ts @@ -15,4 +15,4 @@ export const preetifyResponse = async (c: Context, next: Next) => { endTime(c, "seralize"); c.res = new Response(prettyJson, { headers: { "Content-Type": "text/plain; charset=utf-8" } }); } -}; \ No newline at end of file +}; diff --git a/packages/backend/middleware/rateLimiters.ts b/packages/backend/middleware/rateLimiters.ts index 3c62d7f..c9e444a 100644 --- a/packages/backend/middleware/rateLimiters.ts +++ b/packages/backend/middleware/rateLimiters.ts @@ -24,4 +24,4 @@ export const registerRateLimiter = rateLimiter({ // @ts-expect-error - Known issue: the `c`all` function is not present in @types/ioredis sendCommand: (...args: string[]) => redis.call(...args) }) as unknown as Store -}); \ No newline at end of file +}); diff --git a/packages/backend/routes/404.ts b/packages/backend/routes/404.ts index 910b9f9..4ff883a 100644 --- a/packages/backend/routes/404.ts +++ b/packages/backend/routes/404.ts @@ -7,4 +7,4 @@ export const notFoundRoute = (c: Context) => { }, 404 ); -}; \ No newline at end of file +}; diff --git a/packages/backend/routes/index.ts b/packages/backend/routes/index.ts index 86ebf07..de83d72 100644 --- a/packages/backend/routes/index.ts +++ b/packages/backend/routes/index.ts @@ -1,17 +1,29 @@ -import { rootHandler } from "./root/root.ts"; -import { pingHandler } from "./ping.ts"; -import { getSnapshotsHanlder } from "./snapshots.ts"; -import { registerHandler } from "./user.ts"; -import { videoInfoHandler } from "db/videoInfo.ts"; -import { Hono } from "hono"; -import { Variables } from "hono/types"; +import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "lib/const/singers.ts"; +import { VERSION } from "src/main.ts"; +import { createHandlers } from "src/utils.ts"; -export function configureRoutes(app: Hono<{Variables: Variables }>) { - app.get("/", ...rootHandler); - app.all("/ping", ...pingHandler); - - app.get("/video/:id/snapshots", ...getSnapshotsHanlder); - app.post("/user", ...registerHandler); - - app.get("/video/:id/info", ...videoInfoHandler); -} \ No newline at end of file +export const rootHandler = createHandlers((c) => { + let singer: Singer | Singer[]; + const shouldShowSpecialSinger = Math.random() < 0.016; + if (getSingerForBirthday().length !== 0) { + singer = JSON.parse(JSON.stringify(getSingerForBirthday())) as Singer[]; + for (const s of singer) { + delete s.birthday; + s.message = `祝${s.name}生日快乐~`; + } + } else if (shouldShowSpecialSinger) { + singer = pickSpecialSinger(); + } else { + singer = pickSinger(); + } + return c.json({ + project: { + name: "中V档案馆", + motto: "一起唱吧，心中的歌！" + }, + status: 200, + version: VERSION, + time: Date.now(), + singer: singer + }); +}); diff --git a/packages/backend/routes/ping.ts b/packages/backend/routes/ping.ts deleted file mode 100644 index a84b588..0000000 --- a/packages/backend/routes/ping.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { getClientIP } from "middleware/logger.ts"; -import { createHandlers } from "../src/utils.ts"; -import { VERSION } from "../src/main.ts"; - -export const pingHandler = createHandlers(async (c) => { - const requestHeaders = c.req.raw.headers; - return c.json({ - "message": "pong", - "request": { - "headers": requestHeaders, - "ip": getClientIP(c), - "mode": c.req.raw.mode, - "method": c.req.method, - "query": new URL(c.req.url).searchParams, - "body": await c.req.text(), - "url": c.req.raw.url - }, - "response": { - "time": new Date().getTime(), - "status": 200, - "version": VERSION, - } - }); -}); \ No newline at end of file diff --git a/packages/backend/routes/ping/index.ts b/packages/backend/routes/ping/index.ts new file mode 100644 index 0000000..c1caecb --- /dev/null +++ b/packages/backend/routes/ping/index.ts @@ -0,0 +1,24 @@ +import { getClientIP } from "middleware/logger.ts"; +import { createHandlers } from "src/utils.ts"; +import { VERSION } from "src/main.ts"; + +export const pingHandler = createHandlers(async (c) => { + const requestHeaders = c.req.raw.headers; + return c.json({ + message: "pong", + request: { + headers: requestHeaders, + ip: getClientIP(c), + mode: c.req.raw.mode, + method: c.req.method, + query: new URL(c.req.url).searchParams, + body: await c.req.text(), + url: c.req.raw.url + }, + response: { + time: new Date().getTime(), + status: 200, + version: VERSION + } + }); +}); diff --git a/packages/backend/routes/root/root.ts b/packages/backend/routes/root/root.ts deleted file mode 100644 index d570bce..0000000 --- a/packages/backend/routes/root/root.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "./singers.ts"; -import { VERSION } from "../../src/main.ts"; -import { createHandlers } from "../../src/utils.ts"; - -export const rootHandler = createHandlers((c) => { - let singer: Singer | Singer[]; - const shouldShowSpecialSinger = Math.random() < 0.016; - if (getSingerForBirthday().length !== 0) { - singer = JSON.parse(JSON.stringify(getSingerForBirthday())) as Singer[]; - for (const s of singer) { - delete s.birthday; - s.message = `祝${s.name}生日快乐~`; - } - } else if (shouldShowSpecialSinger) { - singer = pickSpecialSinger(); - } else { - singer = pickSinger(); - } - return c.json({ - project: { - name: "中V档案馆", - motto: "一起唱吧，心中的歌！" - }, - status: 200, - version: VERSION, - time: Date.now(), - singer: singer - }); -}); diff --git a/packages/backend/routes/user/index.ts b/packages/backend/routes/user/index.ts new file mode 100644 index 0000000..d9f7bf7 --- /dev/null +++ b/packages/backend/routes/user/index.ts @@ -0,0 +1 @@ +export * from "./register.ts"; diff --git a/packages/backend/routes/user.ts b/packages/backend/routes/user/register.ts similarity index 99% rename from packages/backend/routes/user.ts rename to packages/backend/routes/user/register.ts index 15cca30..bf1aeeb 100644 --- a/packages/backend/routes/user.ts +++ b/packages/backend/routes/user/register.ts @@ -44,7 +44,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => { const response: StatusResponse = { message: `User '${username}' registered successfully.` - } + }; return c.json(response, 201); } catch (e) { @@ -53,21 +53,21 @@ export const registerHandler = createHandlers(async (c: ContextType) => { message: "Invalid registration data.", errors: e.errors, code: "INVALID_PAYLOAD" - } + }; return c.json>(response, 400); } else if (e instanceof SyntaxError) { const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [e.message], code: "INVALID_FORMAT" - } + }; return c.json>(response, 400); } else { const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [(e as Error).message], code: "UNKNOWN_ERR" - } + }; return c.json>(response, 500); } } diff --git a/packages/backend/db/videoInfo.ts b/packages/backend/routes/video/[id]/info.ts similarity index 84% rename from packages/backend/db/videoInfo.ts rename to packages/backend/routes/video/[id]/info.ts index e9fee88..ad64045 100644 --- a/packages/backend/db/videoInfo.ts +++ b/packages/backend/routes/video/[id]/info.ts @@ -1,15 +1,15 @@ import logger from "@core/log/logger.ts"; import { redis } from "@core/db/redis.ts"; -import { sql } from "./db.ts"; +import { sql } from "../../../db/db.ts"; import { number, ValidationError } from "yup"; -import { createHandlers } from "../src/utils.ts"; +import { createHandlers } from "../../../src/utils.ts"; import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo.ts"; -import { idSchema } from "../routes/snapshots.ts"; +import { idSchema } from "./snapshots.ts"; import { NetSchedulerError } from "@core/net/delegate.ts"; import type { Context } from "hono"; import type { BlankEnv, BlankInput } from "hono/types"; import type { VideoInfoData } from "@core/net/bilibili.d.ts"; -import { startTime, endTime } from 'hono/timing' +import { startTime, endTime } from "hono/timing"; const CACHE_EXPIRATION_SECONDS = 60; @@ -34,7 +34,7 @@ async function insertVideoSnapshot(data: VideoInfoData) { } export const videoInfoHandler = createHandlers(async (c: ContextType) => { - startTime(c, 'parse', 'Parse the request'); + startTime(c, "parse", "Parse the request"); try { const id = await idSchema.validate(c.req.param("id")); let videoId: string | number = id as string; @@ -45,33 +45,33 @@ export const videoInfoHandler = createHandlers(async (c: ContextType) => { } const cacheKey = `cvsa:videoInfo:${videoId}`; - endTime(c, 'parse'); - startTime(c, 'cache', 'Check for cached data'); + endTime(c, "parse"); + startTime(c, "cache", "Check for cached data"); const cachedData = await redis.get(cacheKey); - endTime(c, 'cache'); + endTime(c, "cache"); if (cachedData) { return c.json(JSON.parse(cachedData)); } - startTime(c, 'net', 'Fetch data'); + startTime(c, "net", "Fetch data"); let result: VideoInfoData | number; if (typeof videoId === "number") { result = await getVideoInfo(videoId, "getVideoInfo"); } else { result = await getVideoInfoByBV(videoId, "getVideoInfo"); } - endTime(c, 'net'); + endTime(c, "net"); if (typeof result === "number") { return c.json({ message: "Error fetching video info", code: result }, 500); } - startTime(c, 'db', 'Write data to database'); + startTime(c, "db", "Write data to database"); await redis.setex(cacheKey, CACHE_EXPIRATION_SECONDS, JSON.stringify(result)); await insertVideoSnapshot(result); - endTime(c, 'db'); + endTime(c, "db"); return c.json(result); } catch (e) { if (e instanceof ValidationError) { diff --git a/packages/backend/routes/snapshots.ts b/packages/backend/routes/video/[id]/snapshots.ts similarity index 94% rename from packages/backend/routes/snapshots.ts rename to packages/backend/routes/video/[id]/snapshots.ts index 956a452..a946e75 100644 --- a/packages/backend/routes/snapshots.ts +++ b/packages/backend/routes/video/[id]/snapshots.ts @@ -1,10 +1,10 @@ import type { Context } from "hono"; -import { createHandlers } from "../src/utils.ts"; +import { createHandlers } from "src/utils.ts"; import type { BlankEnv, BlankInput } from "hono/types"; -import { getVideoSnapshots, getVideoSnapshotsByBV } from "../db/videoSnapshot.ts"; +import { getVideoSnapshots, getVideoSnapshotsByBV } from "db/snapshots.ts"; import type { VideoSnapshotType } from "@core/db/schema.d.ts"; import { boolean, mixed, number, object, ValidationError } from "yup"; -import { ErrorResponse } from "../src/schema"; +import { ErrorResponse } from "src/schema"; import { startTime, endTime } from "hono/timing"; const SnapshotQueryParamsSchema = object({ diff --git a/packages/backend/routes/video/index.ts b/packages/backend/routes/video/index.ts new file mode 100644 index 0000000..4c7abb2 --- /dev/null +++ b/packages/backend/routes/video/index.ts @@ -0,0 +1,2 @@ +export * from "./[id]/info"; +export * from "./[id]/snapshots"; diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 06285a7..61ababf 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -1,7 +1,7 @@ import { Hono } from "hono"; import type { TimingVariables } from "hono/timing"; import { startServer } from "./startServer.ts"; -import { configureRoutes } from "routes"; +import { configureRoutes } from "./routing.ts"; import { configureMiddleWares } from "middleware"; import { notFoundRoute } from "routes/404.ts"; diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts new file mode 100644 index 0000000..300e19d --- /dev/null +++ b/packages/backend/src/routing.ts @@ -0,0 +1,16 @@ +import { rootHandler } from "routes"; +import { pingHandler } from "routes/ping"; +import { registerHandler } from "routes/user"; +import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; +import { Hono } from "hono"; +import { Variables } from "hono/types"; + +export function configureRoutes(app: Hono<{ Variables: Variables }>) { + app.get("/", ...rootHandler); + app.all("/ping", ...pingHandler); + + app.get("/video/:id/snapshots", ...getSnapshotsHanlder); + app.post("/user", ...registerHandler); + + app.get("/video/:id/info", ...videoInfoHandler); +} diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index 279a83d..b80ea49 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -1,11 +1,11 @@ type ErrorCode = "INVALID_QUERY_PARAMS" | "UNKNOWN_ERR" | "INVALID_PAYLOAD" | "INVALID_FORMAT" | "BODY_TOO_LARGE"; export interface ErrorResponse { - code: ErrorCode + code: ErrorCode; message: string; errors: E[]; } export interface StatusResponse { message: string; -} \ No newline at end of file +} diff --git a/packages/backend/src/startServer.ts b/packages/backend/src/startServer.ts index 1719393..4df0c07 100644 --- a/packages/backend/src/startServer.ts +++ b/packages/backend/src/startServer.ts @@ -32,7 +32,7 @@ function logStartup(hostname: string, port: number, wasAutoIncremented: boolean, console.log("\nPress Ctrl+C to quit."); } -export async function startServer(app: Hono<{Variables: Variables }>) { +export async function startServer(app: Hono<{ Variables: Variables }>) { const NODE_ENV = process.env.NODE_ENV || "production"; const HOST = process.env.HOST ?? (NODE_ENV === "development" ? "0.0.0.0" : "127.0.0.1"); const PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : undefined; From 8456bb74853053458a8c83ba17907462e53a4ba5 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sat, 10 May 2025 21:48:02 +0800 Subject: [PATCH 03/12] add: route for validation --- bun.lock | 3 +++ packages/backend/middleware/index.ts | 2 +- packages/backend/middleware/rateLimiters.ts | 12 ++++++--- packages/backend/package.json | 1 + .../backend/routes/validation/token/GET.ts | 25 +++++++++++++++++++ .../backend/routes/validation/token/index.ts | 1 + packages/backend/src/routing.ts | 3 +++ 7 files changed, 42 insertions(+), 5 deletions(-) create mode 100644 packages/backend/routes/validation/token/GET.ts create mode 100644 packages/backend/routes/validation/token/index.ts diff --git a/bun.lock b/bun.lock index 5dee910..848c9da 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,7 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", + "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", "yup": "^1.6.1", @@ -845,6 +846,8 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.29.2", "", { "os": "win32", "cpu": "x64" }, "sha512-EdIUW3B2vLuHmv7urfzMI/h2fmlnOQBk1xlsDxkN1tCWKjNFjfLhGxYk8C8mzpSfr+A6jFFIi8fU6LbQGsRWjA=="], + "limiter": ["limiter@3.0.0", "", {}, "sha512-hev7DuXojsTFl2YwyzUJMDnZ/qBDd3yZQLSH3aD4tdL1cqfc3TMnoecEJtWFaQFdErZsKoFMBTxF/FBSkgDbEg=="], + "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], "locate-character": ["locate-character@3.0.0", "", {}, "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="], diff --git a/packages/backend/middleware/index.ts b/packages/backend/middleware/index.ts index 4bd98af..932daf0 100644 --- a/packages/backend/middleware/index.ts +++ b/packages/backend/middleware/index.ts @@ -1,7 +1,7 @@ import { Hono } from "hono"; import { Variables } from "hono/types"; import { bodyLimitForPing } from "./bodyLimits.ts"; -import { pingHandler } from "../routes/ping.ts"; +import { pingHandler } from "routes/ping"; import { registerRateLimiter } from "./rateLimiters.ts"; import { preetifyResponse } from "./preetifyResponse.ts"; import { logger } from "./logger.ts"; diff --git a/packages/backend/middleware/rateLimiters.ts b/packages/backend/middleware/rateLimiters.ts index c9e444a..a065f4c 100644 --- a/packages/backend/middleware/rateLimiters.ts +++ b/packages/backend/middleware/rateLimiters.ts @@ -11,14 +11,18 @@ export const registerRateLimiter = rateLimiter({ limit: 10, standardHeaders: "draft-6", keyGenerator: (c) => { + let ipAddr = crypto.randomUUID() as string; const info = getConnInfo(c as unknown as Context); - if (!info.remote || !info.remote.address) { - return crypto.randomUUID(); + if (info.remote && info.remote.address) { + ipAddr = info.remote.address; + } + const forwardedFor = c.req.header("X-Forwarded-For"); + if (forwardedFor) { + ipAddr = forwardedFor.split(",")[0]; } - const addr = info.remote.address; const path = new URL(c.req.url).pathname; const method = c.req.method; - return `${method}-${path}@${addr}`; + return `${method}-${path}@${ipAddr}`; }, store: new RedisStore({ // @ts-expect-error - Known issue: the `c`all` function is not present in @types/ioredis diff --git a/packages/backend/package.json b/packages/backend/package.json index 16bbc0a..2cc883d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,6 +10,7 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", + "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", "yup": "^1.6.1", diff --git a/packages/backend/routes/validation/token/GET.ts b/packages/backend/routes/validation/token/GET.ts new file mode 100644 index 0000000..ad392ed --- /dev/null +++ b/packages/backend/routes/validation/token/GET.ts @@ -0,0 +1,25 @@ +import { createHandlers } from "src/utils.ts"; + +const DIFFICULTY = 200000; + +const createNewChallenge = async (difficulty: number) => { + const baseURL = process.env["UCAPTCHA_URL"]; + const url = new URL(baseURL); + url.pathname = "/challenge"; + const res = await fetch(url.toString(), { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + difficulty: difficulty, + }) + }); + const data = await res.json(); + return data; +} + +export const createValidationSessionHandler = createHandlers(async (c) => { + const challenge = await createNewChallenge(DIFFICULTY); + return c.json(challenge); +}); diff --git a/packages/backend/routes/validation/token/index.ts b/packages/backend/routes/validation/token/index.ts new file mode 100644 index 0000000..e11870e --- /dev/null +++ b/packages/backend/routes/validation/token/index.ts @@ -0,0 +1 @@ +export * from "./GET.ts"; \ No newline at end of file diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts index 300e19d..783961f 100644 --- a/packages/backend/src/routing.ts +++ b/packages/backend/src/routing.ts @@ -4,6 +4,7 @@ import { registerHandler } from "routes/user"; import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; import { Hono } from "hono"; import { Variables } from "hono/types"; +import { createValidationSessionHandler } from "routes/validation/token"; export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/", ...rootHandler); @@ -13,4 +14,6 @@ export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.post("/user", ...registerHandler); app.get("/video/:id/info", ...videoInfoHandler); + + app.get("/validation/token", ...createValidationSessionHandler) } From 5fb1355346daf7142b186b2b29de9415d890f8b1 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sat, 10 May 2025 22:41:10 +0800 Subject: [PATCH 04/12] ref: rename endpoint to POST /validation/session --- .../routes/validation/{token/GET.ts => session/POST.ts} | 0 packages/backend/routes/validation/session/index.ts | 1 + packages/backend/routes/validation/token/index.ts | 1 - packages/backend/src/routing.ts | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/backend/routes/validation/{token/GET.ts => session/POST.ts} (100%) create mode 100644 packages/backend/routes/validation/session/index.ts delete mode 100644 packages/backend/routes/validation/token/index.ts diff --git a/packages/backend/routes/validation/token/GET.ts b/packages/backend/routes/validation/session/POST.ts similarity index 100% rename from packages/backend/routes/validation/token/GET.ts rename to packages/backend/routes/validation/session/POST.ts diff --git a/packages/backend/routes/validation/session/index.ts b/packages/backend/routes/validation/session/index.ts new file mode 100644 index 0000000..b0f97d2 --- /dev/null +++ b/packages/backend/routes/validation/session/index.ts @@ -0,0 +1 @@ +export * from "./POST.ts"; \ No newline at end of file diff --git a/packages/backend/routes/validation/token/index.ts b/packages/backend/routes/validation/token/index.ts deleted file mode 100644 index e11870e..0000000 --- a/packages/backend/routes/validation/token/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./GET.ts"; \ No newline at end of file diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts index 783961f..9121cf6 100644 --- a/packages/backend/src/routing.ts +++ b/packages/backend/src/routing.ts @@ -4,7 +4,7 @@ import { registerHandler } from "routes/user"; import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; import { Hono } from "hono"; import { Variables } from "hono/types"; -import { createValidationSessionHandler } from "routes/validation/token"; +import { createValidationSessionHandler } from "routes/validation/session"; export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/", ...rootHandler); @@ -15,5 +15,5 @@ export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/video/:id/info", ...videoInfoHandler); - app.get("/validation/token", ...createValidationSessionHandler) + app.post("/validation/session", ...createValidationSessionHandler) } From 137c19d74e7693f50a4108b87567ebd0b48eb6c7 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 11 May 2025 01:50:02 +0800 Subject: [PATCH 05/12] ref: move from sliding window to token bucket in rate limiter --- packages/core/mq/rateLimiter.ts | 106 +++++++++++++++++------------- packages/core/mq/slidingWindow.ts | 51 -------------- packages/core/mq/tokenBucket.ts | 84 +++++++++++++++++++++++ packages/core/net/delegate.ts | 17 +++-- 4 files changed, 151 insertions(+), 107 deletions(-) delete mode 100644 packages/core/mq/slidingWindow.ts create mode 100644 packages/core/mq/tokenBucket.ts diff --git a/packages/core/mq/rateLimiter.ts b/packages/core/mq/rateLimiter.ts index ac42748..96f208c 100644 --- a/packages/core/mq/rateLimiter.ts +++ b/packages/core/mq/rateLimiter.ts @@ -1,56 +1,68 @@ -import type { SlidingWindow } from "./slidingWindow.ts"; +import { TokenBucket } from "./tokenBucket.ts"; export interface RateLimiterConfig { - window: SlidingWindow; - max: number; + duration: number; + max: number; } export class RateLimiter { - private readonly configs: RateLimiterConfig[]; - private readonly configEventNames: string[]; + private configs: RateLimiterConfig[] = []; + private buckets: TokenBucket[] = []; + private identifierFn: (configIndex: number) => string; - /* - * @param name The name of the rate limiter - * @param configs The configuration of the rate limiter, containing: - * - window: The sliding window to use - * - max: The maximum number of events allowed in the window - */ - constructor(name: string, configs: RateLimiterConfig[]) { - this.configs = configs; - this.configEventNames = configs.map((_, index) => `${name}_config_${index}`); - } + /* + * @param name The name of the rate limiter + * @param configs The configuration of the rate limiter, containing: + * - tokenBucket: The token bucket instance + * - max: The maximum number of tokens allowed per operation + */ + constructor( + name: string, + configs: RateLimiterConfig[], + identifierFn?: (configIndex: number) => string + ) { + this.configs = configs; + this.identifierFn = identifierFn || ((index) => `${name}_config_${index}`); + for (let i = 0; i < configs.length; i++) { + const config = configs[i]; + const bucket = new TokenBucket({ + capacity: config.max, + rate: config.max / config.duration, + identifier: this.identifierFn(i), + }) + this.buckets.push(bucket); + } + } - /* - * Check if the event has reached the rate limit - */ - async getAvailability(): Promise { - for (let i = 0; i < this.configs.length; i++) { - const config = this.configs[i]; - const eventName = this.configEventNames[i]; - const count = await config.window.count(eventName); - if (count >= config.max) { - return false; - } - } - return true; - } + /* + * Check if the event has reached the rate limit + */ + async getAvailability(): Promise { + for (let i = 0; i < this.configs.length; i++) { + const remaining = await this.buckets[i].getRemainingTokens(); - /* - * Trigger an event in the rate limiter - */ - async trigger(): Promise { - for (let i = 0; i < this.configs.length; i++) { - const config = this.configs[i]; - const eventName = this.configEventNames[i]; - await config.window.event(eventName); - } - } + if (remaining === null) { + return false; // Rate limit exceeded + } + } + return true; + } - async clear(): Promise { - for (let i = 0; i < this.configs.length; i++) { - const config = this.configs[i]; - const eventName = this.configEventNames[i]; - await config.window.clear(eventName); - } - } -} + /* + * Trigger an event in the rate limiter + */ + async trigger(): Promise { + for (let i = 0; i < this.configs.length; i++) { + await this.buckets[i].consume(1); + } + } + + /* + * Clear all buckets for all configurations + */ + async clear(): Promise { + for (let i = 0; i < this.configs.length; i++) { + await this.buckets[i].reset(); + } + } +} \ No newline at end of file diff --git a/packages/core/mq/slidingWindow.ts b/packages/core/mq/slidingWindow.ts deleted file mode 100644 index 457303c..0000000 --- a/packages/core/mq/slidingWindow.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { Redis } from "ioredis"; - -export class SlidingWindow { - private redis: Redis; - private readonly windowSize: number; - - /* - * Create a new sliding window - * @param redisClient The Redis client used to store the data - * @param windowSize The size of the window in seconds - */ - constructor(redisClient: Redis, windowSize: number) { - this.redis = redisClient; - this.windowSize = windowSize * 1000; - } - - /* - * Trigger an event in the sliding window - * @param eventName The name of the event - */ - async event(eventName: string): Promise { - const now = Date.now(); - const key = `cvsa:sliding_window:${eventName}`; - - const uniqueMember = `${now}-${Math.random()}`; - // Add current timestamp to an ordered set - await this.redis.zadd(key, now, uniqueMember); - - // Remove timestamps outside the window - await this.redis.zremrangebyscore(key, 0, now - this.windowSize); - } - - /* - * Count the number of events in the sliding window - * @param eventName The name of the event - */ - async count(eventName: string): Promise { - const key = `cvsa:sliding_window:${eventName}`; - const now = Date.now(); - - // Remove timestamps outside the window - await this.redis.zremrangebyscore(key, 0, now - this.windowSize); - // Get the number of timestamps in the window - return this.redis.zcard(key); - } - - clear(eventName: string): Promise { - const key = `cvsa:sliding_window:${eventName}`; - return this.redis.del(key); - } -} diff --git a/packages/core/mq/tokenBucket.ts b/packages/core/mq/tokenBucket.ts new file mode 100644 index 0000000..8e94db9 --- /dev/null +++ b/packages/core/mq/tokenBucket.ts @@ -0,0 +1,84 @@ +import { redis } from "@core/db/redis"; +import { SECOND } from "@core/const/time"; + +interface TokenBucketOptions { + capacity: number; + rate: number; + identifier: string; + keyPrefix?: string; +} + +export class TokenBucket { + private readonly capacity: number; + private readonly rate: number; + private readonly keyPrefix: string; + private readonly identifier: string; + + constructor(options: TokenBucketOptions) { + if (options.capacity <= 0 || options.rate <= 0) { + throw new Error("Capacity and rate must be greater than zero."); + } + + this.capacity = options.capacity; + this.rate = options.rate; + this.identifier = options.identifier; + this.keyPrefix = options.keyPrefix || "cvsa:token_bucket:"; + } + + getKey(): string { + return `${this.keyPrefix}${this.identifier}`; + } + + /** + * Try to consume a specified number of tokens + * @param count The number of tokens to be consumed + * @returns If consumption is successful, returns the number of remaining tokens; otherwise returns null + */ + public async consume(count: number): Promise { + const key = this.getKey(); + const now = Math.floor(Date.now() / SECOND); + + const script = ` + local tokens_key = KEYS[1] + local last_refilled_key = KEYS[2] + local now = tonumber(ARGV[1]) + local count = tonumber(ARGV[2]) + local capacity = tonumber(ARGV[3]) + local rate = tonumber(ARGV[4]) + + local last_refilled = tonumber(redis.call('GET', last_refilled_key)) or now + local current_tokens = tonumber(redis.call('GET', tokens_key)) or capacity + + local elapsed = now - last_refilled + local new_tokens = elapsed * rate + current_tokens = math.min(capacity, current_tokens + new_tokens) + + if current_tokens >= count then + current_tokens = current_tokens - count + redis.call('SET', tokens_key, current_tokens) + redis.call('SET', last_refilled_key, now) + return current_tokens + else + return nil + end + `; + + const keys = [`${key}:tokens`, `${key}:last_refilled`]; + const args = [now, count, this.capacity, this.rate]; + + const result = await redis.eval(script, keys.length, ...keys, ...args); + + return result as number | null; + } + + public async getRemainingTokens(): Promise { + const key = this.getKey(); + const tokens = await redis.get(`${key}:tokens`); + return Number(tokens) || this.capacity; + } + + public async reset(): Promise { + const key = this.getKey(); + await redis.del(`${key}:tokens`, `${key}:last_refilled`); + } +} diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index e7ca3f1..e1518c0 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,6 +1,5 @@ import logger from "@core/log/logger.ts"; import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts"; -import { SlidingWindow } from "mq/slidingWindow.ts"; import { redis } from "db/redis.ts"; import { ReplyError } from "ioredis"; import { SECOND } from "../const/time.ts"; @@ -316,37 +315,37 @@ class NetworkDelegate { const networkDelegate = new NetworkDelegate(); const videoInfoRateLimiterConfig: RateLimiterConfig[] = [ { - window: new SlidingWindow(redis, 0.3), + duration: 0.3, max: 1, }, { - window: new SlidingWindow(redis, 3), + duration: 3, max: 5, }, { - window: new SlidingWindow(redis, 30), + duration: 30, max: 30, }, { - window: new SlidingWindow(redis, 2 * 60), + duration: 2 * 60, max: 50, }, ]; const biliLimiterConfig: RateLimiterConfig[] = [ { - window: new SlidingWindow(redis, 1), + duration: 1, max: 6, }, { - window: new SlidingWindow(redis, 5), + duration: 5, max: 20, }, { - window: new SlidingWindow(redis, 30), + duration: 30, max: 100, }, { - window: new SlidingWindow(redis, 5 * 60), + duration: 5 * 60, max: 200, }, ]; From a063f2401bef9490e6ecf6579e3d4a0b671e75d5 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 11 May 2025 19:14:19 +0800 Subject: [PATCH 06/12] update: rename the captcha endpoint --- .gitignore | 4 +- .../backend/routes/captcha/[id]/result/GET.ts | 37 +++++++++++++++++++ packages/backend/routes/captcha/index.ts | 2 + .../{validation => captcha}/session/POST.ts | 9 ++--- .../routes/validation/session/index.ts | 1 - packages/backend/src/routing.ts | 5 ++- packages/backend/src/schema.d.ts | 2 +- 7 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 packages/backend/routes/captcha/[id]/result/GET.ts create mode 100644 packages/backend/routes/captcha/index.ts rename packages/backend/routes/{validation => captcha}/session/POST.ts (69%) delete mode 100644 packages/backend/routes/validation/session/index.ts diff --git a/.gitignore b/.gitignore index e2ff74a..c0bff53 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,6 @@ redis/ dist/ build/ -docker-compose.yml \ No newline at end of file +docker-compose.yml + +ucaptcha-config.yaml \ No newline at end of file diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts new file mode 100644 index 0000000..aa1a975 --- /dev/null +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -0,0 +1,37 @@ +import { Context } from "hono"; +import { Bindings, BlankEnv, BlankInput } from "hono/types"; +import { ErrorResponse } from "src/schema"; +import { createHandlers } from "src/utils.ts"; + +const getChallengeVerificationResult = async (id: string, ans: string) => { + const baseURL = process.env["UCAPTCHA_URL"]; + const url = new URL(baseURL); + url.pathname = `/challenge/${id}/validation`; + const res = await fetch(url.toString(), { + method: "POST", + headers: { + "Content-Type": "application/json" + }, + body: JSON.stringify({ + y: ans + }) + }); + return res; +}; + + +export const verifyChallengeHandler = createHandlers( + async (c: Context) => { + const id = c.req.param("id"); + const ans = c.req.query("ans"); + if (!ans) { + const response: ErrorResponse = { + message: "Missing required query parameter: ans", + code: "INVALID_QUERY_PARAMS" + }; + return c.json>(response, 400); + } + const res = await getChallengeVerificationResult(id, ans); + return res; + } +); diff --git a/packages/backend/routes/captcha/index.ts b/packages/backend/routes/captcha/index.ts new file mode 100644 index 0000000..fa6d476 --- /dev/null +++ b/packages/backend/routes/captcha/index.ts @@ -0,0 +1,2 @@ +export * from "./session/POST.ts"; +export * from "./[id]/result/GET.ts"; \ No newline at end of file diff --git a/packages/backend/routes/validation/session/POST.ts b/packages/backend/routes/captcha/session/POST.ts similarity index 69% rename from packages/backend/routes/validation/session/POST.ts rename to packages/backend/routes/captcha/session/POST.ts index ad392ed..63b9331 100644 --- a/packages/backend/routes/validation/session/POST.ts +++ b/packages/backend/routes/captcha/session/POST.ts @@ -15,11 +15,10 @@ const createNewChallenge = async (difficulty: number) => { difficulty: difficulty, }) }); - const data = await res.json(); - return data; + return res; } -export const createValidationSessionHandler = createHandlers(async (c) => { - const challenge = await createNewChallenge(DIFFICULTY); - return c.json(challenge); +export const createCaptchaSessionHandler = createHandlers(async (_c) => { + const res = await createNewChallenge(DIFFICULTY); + return res; }); diff --git a/packages/backend/routes/validation/session/index.ts b/packages/backend/routes/validation/session/index.ts deleted file mode 100644 index b0f97d2..0000000 --- a/packages/backend/routes/validation/session/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./POST.ts"; \ No newline at end of file diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts index 9121cf6..a56ed73 100644 --- a/packages/backend/src/routing.ts +++ b/packages/backend/src/routing.ts @@ -4,7 +4,7 @@ import { registerHandler } from "routes/user"; import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; import { Hono } from "hono"; import { Variables } from "hono/types"; -import { createValidationSessionHandler } from "routes/validation/session"; +import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha"; export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/", ...rootHandler); @@ -15,5 +15,6 @@ export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/video/:id/info", ...videoInfoHandler); - app.post("/validation/session", ...createValidationSessionHandler) + app.post("/captcha/session", ...createCaptchaSessionHandler); + app.get("/captcha/:id/result", ...verifyChallengeHandler); } diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index b80ea49..db279b7 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -3,7 +3,7 @@ type ErrorCode = "INVALID_QUERY_PARAMS" | "UNKNOWN_ERR" | "INVALID_PAYLOAD" | "I export interface ErrorResponse { code: ErrorCode; message: string; - errors: E[]; + errors?: E[]; } export interface StatusResponse { From 1633e56b1e559f485cb1dc458624de40a2aea2b6 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 11 May 2025 19:40:24 +0800 Subject: [PATCH 07/12] update: returns JWT in the /captcha/:id/result endpoint --- bun.lock | 3 + packages/backend/package.json | 1 + .../backend/routes/captcha/[id]/result/GET.ts | 70 ++++++++++++++++--- packages/backend/routes/user/register.ts | 2 +- .../backend/routes/video/[id]/snapshots.ts | 2 +- packages/backend/src/schema.d.ts | 11 ++- 6 files changed, 76 insertions(+), 13 deletions(-) diff --git a/bun.lock b/bun.lock index 848c9da..f5d6db7 100644 --- a/bun.lock +++ b/bun.lock @@ -19,6 +19,7 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", + "jose": "^6.0.11", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", @@ -814,6 +815,8 @@ "jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="], + "jose": ["jose@6.0.11", "", {}, "sha512-QxG7EaliDARm1O1S8BGakqncGT9s25bKL1WSf6/oa17Tkqwi8D2ZNglqCF+DsYF88/rV66Q/Q2mFAy697E1DUg=="], + "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], diff --git a/packages/backend/package.json b/packages/backend/package.json index 2cc883d..aede4e7 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,6 +10,7 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", + "jose": "^6.0.11", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts index aa1a975..6a507da 100644 --- a/packages/backend/routes/captcha/[id]/result/GET.ts +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -2,6 +2,12 @@ import { Context } from "hono"; import { Bindings, BlankEnv, BlankInput } from "hono/types"; import { ErrorResponse } from "src/schema"; import { createHandlers } from "src/utils.ts"; +import * as jose from "jose"; + +interface CaptchaResponse { + success: boolean; + error?: string; +} const getChallengeVerificationResult = async (id: string, ans: string) => { const baseURL = process.env["UCAPTCHA_URL"]; @@ -19,19 +25,63 @@ const getChallengeVerificationResult = async (id: string, ans: string) => { return res; }; - export const verifyChallengeHandler = createHandlers( async (c: Context) => { const id = c.req.param("id"); const ans = c.req.query("ans"); - if (!ans) { - const response: ErrorResponse = { - message: "Missing required query parameter: ans", - code: "INVALID_QUERY_PARAMS" - }; - return c.json>(response, 400); - } - const res = await getChallengeVerificationResult(id, ans); - return res; + if (!ans) { + const response: ErrorResponse = { + message: "Missing required query parameter: ans", + code: "INVALID_QUERY_PARAMS" + }; + return c.json>(response, 400); + } + const res = await getChallengeVerificationResult(id, ans); + const data: CaptchaResponse = await res.json(); + if (data.error && res.status === 404) { + const response: ErrorResponse = { + message: data.error, + code: "ENTITY_NOT_FOUND" + }; + return c.json>(response, 401); + } else if (data.error && res.status === 400) { + const response: ErrorResponse = { + message: data.error, + code: "INVALID_QUERY_PARAMS" + }; + return c.json>(response, 400); + } else if (data.error) { + const response: ErrorResponse = { + message: data.error, + code: "UNKNOWN_ERROR" + }; + return c.json>(response, 500); + } + if (!data.success) { + const response: ErrorResponse = { + message: "Incorrect answer", + code: "INVALID_CREDENTIALS" + }; + return c.json>(response, 401); + } + + const secret = process.env["JWT_SECRET"]; + if (!secret) { + const response: ErrorResponse = { + message: "JWT_SECRET is not set", + code: "SERVER_ERROR" + }; + return c.json>(response, 500); + } + const jwtSecret = new TextEncoder().encode(secret); + const alg = "HS256"; + + const jwt = await new jose.SignJWT() + .setProtectedHeader({ alg }) + .setIssuedAt() + .sign(jwtSecret); + return c.json({ + token: jwt + }); } ); diff --git a/packages/backend/routes/user/register.ts b/packages/backend/routes/user/register.ts index bf1aeeb..ac8200d 100644 --- a/packages/backend/routes/user/register.ts +++ b/packages/backend/routes/user/register.ts @@ -66,7 +66,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => { const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [(e as Error).message], - code: "UNKNOWN_ERR" + code: "UNKNOWN_ERROR" }; return c.json>(response, 500); } diff --git a/packages/backend/routes/video/[id]/snapshots.ts b/packages/backend/routes/video/[id]/snapshots.ts index a946e75..e738e50 100644 --- a/packages/backend/routes/video/[id]/snapshots.ts +++ b/packages/backend/routes/video/[id]/snapshots.ts @@ -96,7 +96,7 @@ export const getSnapshotsHanlder = createHandlers(async (c: ContextType) => { return c.json>(response, 400); } else { const response: ErrorResponse = { - code: "UNKNOWN_ERR", + code: "UNKNOWN_ERROR", message: "Unhandled error", errors: [e] }; diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index db279b7..51f54fa 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -1,4 +1,13 @@ -type ErrorCode = "INVALID_QUERY_PARAMS" | "UNKNOWN_ERR" | "INVALID_PAYLOAD" | "INVALID_FORMAT" | "BODY_TOO_LARGE"; +type ErrorCode = + | "INVALID_QUERY_PARAMS" + | "UNKNOWN_ERROR" + | "INVALID_PAYLOAD" + | "INVALID_FORMAT" + | "BODY_TOO_LARGE" + | "UNAUTHORIZED" + | "INVALID_CREDENTIALS" + | "ENTITY_NOT_FOUND" + | "SERVER_ERROR"; export interface ErrorResponse { code: ErrorCode; From b18b45078fd1f5c0f23184828df142aad6457c84 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sat, 17 May 2025 02:15:05 +0800 Subject: [PATCH 08/12] add: tokenID added to JWT in endpoint GET /captcha/:id/result --- .../backend/routes/captcha/[id]/result/GET.ts | 16 +- packages/core/deno.json | 17 --- packages/core/lib/randomID.ts | 142 ++++++++++++++++++ packages/{crawler => core}/mq/lockManager.ts | 2 +- packages/core/net/delegate.ts | 3 +- packages/core/package.json | 27 ++-- packages/core/test/lib/randomID.test.ts | 80 ++++++++++ packages/crawler/mq/exec/archiveSnapshots.ts | 2 +- packages/crawler/mq/exec/classifyVideo.ts | 2 +- .../mq/exec/dispatchRegularSnapshots.ts | 2 +- packages/crawler/mq/exec/snapshotVideo.ts | 2 +- packages/crawler/src/filterWorker.ts | 2 +- packages/crawler/src/worker.ts | 2 +- 13 files changed, 256 insertions(+), 43 deletions(-) delete mode 100644 packages/core/deno.json create mode 100644 packages/core/lib/randomID.ts rename packages/{crawler => core}/mq/lockManager.ts (96%) create mode 100644 packages/core/test/lib/randomID.test.ts diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts index 6a507da..cffab6d 100644 --- a/packages/backend/routes/captcha/[id]/result/GET.ts +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -1,11 +1,14 @@ import { Context } from "hono"; -import { Bindings, BlankEnv, BlankInput } from "hono/types"; +import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; import { createHandlers } from "src/utils.ts"; import * as jose from "jose"; +import { generateRandomId } from "@core/lib/randomID.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; interface CaptchaResponse { success: boolean; + difficulty?: number; error?: string; } @@ -13,7 +16,7 @@ const getChallengeVerificationResult = async (id: string, ans: string) => { const baseURL = process.env["UCAPTCHA_URL"]; const url = new URL(baseURL); url.pathname = `/challenge/${id}/validation`; - const res = await fetch(url.toString(), { + return await fetch(url.toString(), { method: "POST", headers: { "Content-Type": "application/json" @@ -22,11 +25,10 @@ const getChallengeVerificationResult = async (id: string, ans: string) => { y: ans }) }); - return res; }; export const verifyChallengeHandler = createHandlers( - async (c: Context) => { + async (c: Context) => { const id = c.req.param("id"); const ans = c.req.query("ans"); if (!ans) { @@ -76,7 +78,11 @@ export const verifyChallengeHandler = createHandlers( const jwtSecret = new TextEncoder().encode(secret); const alg = "HS256"; - const jwt = await new jose.SignJWT() + + const tokenID = generateRandomId(10); + const EXPIRE_FIVE_MINUTES = 300; + await lockManager.acquireLock(tokenID, EXPIRE_FIVE_MINUTES); + const jwt = await new jose.SignJWT({ difficulty: data.difficulty!, id: tokenID }) .setProtectedHeader({ alg }) .setIssuedAt() .sign(jwtSecret); diff --git a/packages/core/deno.json b/packages/core/deno.json deleted file mode 100644 index b1027e9..0000000 --- a/packages/core/deno.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@cvsa/core", - "exports": "./main.ts", - "imports": { - "ioredis": "npm:ioredis", - "log/": "./log/", - "db/": "./db/", - "$std/": "https://deno.land/std@0.216.0/", - "mq/": "./mq/", - "chalk": "npm:chalk", - "winston": "npm:winston", - "logform": "npm:logform", - "@core/": "./", - "child_process": "node:child_process", - "util": "node:util" - } -} diff --git a/packages/core/lib/randomID.ts b/packages/core/lib/randomID.ts new file mode 100644 index 0000000..477fe4a --- /dev/null +++ b/packages/core/lib/randomID.ts @@ -0,0 +1,142 @@ +const getSecureRandomInt = (max: number): number => { + const array = new Uint32Array(1); + crypto.getRandomValues(array); + // Using modulo bias is technically present, but negligible here because the space (56) is tiny compared to 2^32. + return array[0] % max; +} + + +/** + * Generates a random ID with characteristics similar to UUIDv7, + * incorporating a timestamp prefix for sortability and a random suffix, + * using a customizable length and a specific character set. + * + * This function aims for sortability by placing a time-based component at the beginning, + * similar to UUIDv7, while allowing a variable total length and using a character set + * designed to avoid visually similar characters. + * + * The character set includes uppercase and lowercase letters and numbers, + * excluding visually similar characters (0, O, I, l, 1). + * + * **Length Reference**: + * + * With a collision probability of **0.1%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **10**: 1.8 IDs / ms or 1,844 QPS + * - **12**: 27 IDs / ms or 26,998 QPS + * - **16**: 5784 IDs / ms or 5,784,295 QPS + * + * With a collision probability of **0.001%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **11**: 1.5 IDs / ms or 1,520 QPS + * - **14**: 85 IDs / ms or 85,124 QPS + * - **16**: 1246 IDs / ms or 1,245,983 QPS + * + * With a collision probability of **0.00001%**, + * the maximum ID generation rate per millisecond for the following lengths is: + * - **14**: 18 IDs / ms or 18,339 QPS + * - **15**: 70 IDs / ms or 70,164 QPS + * - **16**: 1246 IDs / ms or 268,438 QPS + * + * The formula: max_qps = 1000 * (2 * (56**(length - 8)) * -log(1 - prob))**(1/3) + * + * @param length The desired total length of the ID. Must be at least 8. + * @returns A sortable random ID string of the specified length. + * @throws Error if the requested length is less than the minimum required for the timestamp prefix (8). + */ +export function generateRandomId(length: number): string { + // Character set excluding 0, O, I, l, 1 + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters + const base = allowedChars.length; // 56 + const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part to ensure sortability + + if (length < TIMESTAMP_PREFIX_LENGTH) { + throw new Error(`Length must be at least ${TIMESTAMP_PREFIX_LENGTH} to include the timestamp prefix.`); + } + + // --- Generate Timestamp Prefix --- + const timestamp = Date.now(); // Milliseconds since epoch (Unix time) + let timestampBaseString = ""; + let tempTimestamp = timestamp; + const firstChar = allowedChars[0]; // Character for padding ('a') + + // Convert timestamp to a base-56 string + // We process from the least significant "digit" to the most significant + while (tempTimestamp > 0) { + timestampBaseString = allowedChars[tempTimestamp % base] + timestampBaseString; + tempTimestamp = Math.floor(tempTimestamp / base); + } + + // Pad the timestamp string at the beginning to ensure a fixed length. + // This is crucial for chronological sortability of the generated IDs. + while (timestampBaseString.length < TIMESTAMP_PREFIX_LENGTH) { + timestampBaseString = firstChar + timestampBaseString; + } + + // Although highly unlikely with an 8-character prefix using base 56 for current timestamps, + // this would truncate if the timestamp string somehow exceeded the prefix length. + if (timestampBaseString.length > TIMESTAMP_PREFIX_LENGTH) { + timestampBaseString = timestampBaseString.substring(timestampBaseString.length - TIMESTAMP_PREFIX_LENGTH); + } + + // --- Generate Random Suffix --- + const randomLength = length - TIMESTAMP_PREFIX_LENGTH; + let randomSuffix = ""; + const allowedCharsLength = allowedChars.length; + + for (let i = 0; i < randomLength; i++) { + const randomIndex = getSecureRandomInt(allowedCharsLength); + randomSuffix += allowedChars[randomIndex]; + } + + // --- Concatenate and Return --- + return timestampBaseString + randomSuffix; +} + +/** + * Decodes the timestamp (in milliseconds since epoch) from an ID + * generated by a function that uses a fixed-length timestamp prefix + * encoded in a specific base and character set (like the previous example). + * + * It extracts the timestamp prefix and converts it back from the + * custom base-56 encoding to a number. + * + * @param id The ID string containing the timestamp prefix. + * @returns The timestamp in milliseconds since epoch. + * @throws Error if the ID is too short or contains invalid characters in the timestamp prefix. + */ +export function decodeTimestampFromId(id: string): number { + // Character set must match the encoding function used to generate the ID + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters + const base = allowedChars.length; // 56 + const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part + + if (id.length < TIMESTAMP_PREFIX_LENGTH) { + throw new Error( + `ID must be at least ${TIMESTAMP_PREFIX_LENGTH} characters long to contain a timestamp prefix.` + ); + } + + // Extract the timestamp prefix from the beginning of the ID string + const timestampPrefix = id.substring(0, TIMESTAMP_PREFIX_LENGTH); + let timestamp = 0; + + // Convert the base-56 timestamp string back to a number + // Iterate through the prefix characters from left to right (most significant 'digit') + for (let i = 0; i < timestampPrefix.length; i++) { + const char = timestampPrefix[i]; + // Find the index (value) of the character in the allowed character set + const charIndex = allowedChars.indexOf(char); + + if (charIndex === -1) { + // If a character is not in the allowed set, the ID is likely invalid + throw new Error(`Invalid character "${char}" found in timestamp prefix.`); + } + + // Standard base conversion: accumulate the value + // For each digit, multiply the current total by the base and add the digit's value + timestamp = timestamp * base + charIndex; + } + + return timestamp; +} diff --git a/packages/crawler/mq/lockManager.ts b/packages/core/mq/lockManager.ts similarity index 96% rename from packages/crawler/mq/lockManager.ts rename to packages/core/mq/lockManager.ts index fefae7b..129001a 100644 --- a/packages/crawler/mq/lockManager.ts +++ b/packages/core/mq/lockManager.ts @@ -1,5 +1,5 @@ import { Redis } from "ioredis"; -import { redis } from "../../core/db/redis.ts"; +import { redis } from "@core/db/redis.ts"; class LockManager { private redis: Redis; diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index e1518c0..6d7d6d7 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,8 +1,7 @@ import logger from "@core/log/logger.ts"; import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts"; -import { redis } from "db/redis.ts"; import { ReplyError } from "ioredis"; -import { SECOND } from "../const/time.ts"; +import { SECOND } from "@core/const/time.ts"; import { spawn, SpawnOptions } from "child_process"; export function spawnPromise( diff --git a/packages/core/package.json b/packages/core/package.json index a11ede0..9c291cb 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,13 +1,16 @@ { - "name": "core", - "dependencies": { - "chalk": "^5.4.1", - "ioredis": "^5.6.1", - "logform": "^2.7.0", - "postgres": "^3.4.5", - "winston": "^3.17.0" - }, - "devDependencies": { - "@types/ioredis": "^5.0.0" - } -} \ No newline at end of file + "name": "core", + "scripts": { + "test": "bun --env-file=.env.test run vitest" + }, + "dependencies": { + "chalk": "^5.4.1", + "ioredis": "^5.6.1", + "logform": "^2.7.0", + "postgres": "^3.4.5", + "winston": "^3.17.0" + }, + "devDependencies": { + "@types/ioredis": "^5.0.0" + } +} diff --git a/packages/core/test/lib/randomID.test.ts b/packages/core/test/lib/randomID.test.ts new file mode 100644 index 0000000..71b859c --- /dev/null +++ b/packages/core/test/lib/randomID.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { generateRandomId, decodeTimestampFromId } from "@core/lib/randomID.ts"; + +describe("generateRandomId", () => { + it("should throw an error if the requested length is less than 8", () => { + expect(() => generateRandomId(7)).toThrowError("Length must be at least 8 to include the timestamp prefix."); + }); + + it("should generate an ID of the specified length", () => { + const length = 15; + const id = generateRandomId(length); + expect(id).toHaveLength(length); + }); + + it("should generate an ID with a timestamp prefix of length 8", () => { + const id = generateRandomId(12); + expect(id).toHaveProperty("substring"); + expect(id).toHaveProperty("length"); + expect(id.length).toBeGreaterThanOrEqual(8); + }); + + it("should generate an ID containing only allowed characters", () => { + const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; + const id = generateRandomId(20); + for (const char of id) { + expect(allowedChars).toContain(char); + } + }); + + it("should generate IDs that are sortable by creation time", () => { + const id1 = generateRandomId(10); + // Simulate a slight delay to ensure different timestamps + return new Promise((resolve) => { + setTimeout(() => { + const id2 = generateRandomId(10); + expect(id2 >= id1).toBe(true); + resolve(null); + }, 10); + }); + }); +}); + +describe("decodeTimestampFromId", () => { + it("should throw an error if the ID length is less than 8", () => { + expect(() => decodeTimestampFromId("abcdefg")).toThrowError( + "ID must be at least 8 characters long to contain a timestamp prefix." + ); + }); + + it("should throw an error if the timestamp prefix contains invalid characters", () => { + const invalidId = "0bcdefghijk"; + expect(() => decodeTimestampFromId(invalidId)).toThrowError('Invalid character "0" found in timestamp prefix.'); + }); + + it("should correctly decode the timestamp from a generated ID", () => { + const now = Date.now(); + // Mock Date.now to control the timestamp for testing + const originalDateNow = Date.now; + global.Date.now = () => now; + const id = generateRandomId(16); + global.Date.now = originalDateNow; // Restore original Date.now + + const decodedTimestamp = decodeTimestampFromId(id); + // Allow a small margin for potential timing differences in test execution + expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); + expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); + }); + + it("should correctly decode the timestamp even with a longer ID", () => { + const now = Date.now(); + const originalDateNow = Date.now; + global.Date.now = () => now; + const id = generateRandomId(20); + global.Date.now = originalDateNow; + + const decodedTimestamp = decodeTimestampFromId(id); + expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); + expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); + }); +}); diff --git a/packages/crawler/mq/exec/archiveSnapshots.ts b/packages/crawler/mq/exec/archiveSnapshots.ts index 91411a9..c667416 100644 --- a/packages/crawler/mq/exec/archiveSnapshots.ts +++ b/packages/crawler/mq/exec/archiveSnapshots.ts @@ -1,7 +1,7 @@ import { Job } from "bullmq"; import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getLatestVideoSnapshot } from "db/snapshot.ts"; import { HOUR, MINUTE } from "@core/const/time.ts"; import { sql } from "@core/db/dbNew"; diff --git a/packages/crawler/mq/exec/classifyVideo.ts b/packages/crawler/mq/exec/classifyVideo.ts index 622eeed..6c3d37c 100644 --- a/packages/crawler/mq/exec/classifyVideo.ts +++ b/packages/crawler/mq/exec/classifyVideo.ts @@ -3,7 +3,7 @@ import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from " import Akari from "ml/akari.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { aidExistsInSongs } from "db/songs.ts"; import { insertIntoSongs } from "mq/task/collectSongs.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; diff --git a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts index 5c1652f..49a7893 100644 --- a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts @@ -4,7 +4,7 @@ import { truncate } from "utils/truncate.ts"; import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; import { HOUR, MINUTE, WEEK } from "@core/const/time.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts"; import { sql } from "@core/db/dbNew.ts"; diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts index 261b78b..59f05db 100644 --- a/packages/crawler/mq/exec/snapshotVideo.ts +++ b/packages/crawler/mq/exec/snapshotVideo.ts @@ -2,7 +2,7 @@ import { Job } from "bullmq"; import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts"; import logger from "@core/log/logger.ts"; import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts"; import { insertVideoSnapshot } from "mq/task/getVideoStats.ts"; import { getSongsPublihsedAt } from "db/songs.ts"; diff --git a/packages/crawler/src/filterWorker.ts b/packages/crawler/src/filterWorker.ts index 7ba2de6..ab85cba 100644 --- a/packages/crawler/src/filterWorker.ts +++ b/packages/crawler/src/filterWorker.ts @@ -3,7 +3,7 @@ import { redis } from "@core/db/redis.ts"; import logger from "@core/log/logger.ts"; import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts"; import { WorkerError } from "mq/schema.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import Akari from "ml/akari.ts"; const shutdown = async (signal: string) => { diff --git a/packages/crawler/src/worker.ts b/packages/crawler/src/worker.ts index ca644df..534f488 100644 --- a/packages/crawler/src/worker.ts +++ b/packages/crawler/src/worker.ts @@ -14,7 +14,7 @@ import { } from "mq/exec/executors.ts"; import { redis } from "@core/db/redis.ts"; import logger from "@core/log/logger.ts"; -import { lockManager } from "mq/lockManager.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; import { WorkerError } from "mq/schema.ts"; const releaseLockForJob = async (name: string) => { From 7786d66dbb4c31d9b741134c41d0dcd5f9569e64 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 18 May 2025 00:33:43 +0800 Subject: [PATCH 09/12] add: complete guarding under uCaptcha --- .idea/cvsa.iml | 2 + deno.json | 16 -- packages/backend/db/config.ts | 46 ------ packages/backend/db/db.ts | 5 - packages/backend/db/snapshots.ts | 2 +- .../backend/lib/auth/captchaDifficulty.ts | 57 +++++++ packages/backend/lib/auth/getJWTsecret.ts | 13 ++ packages/backend/middleware/captcha.ts | 112 +++++++++++++ packages/backend/middleware/index.ts | 2 + packages/backend/middleware/rateLimiters.ts | 34 ++-- packages/backend/package.json | 1 - .../backend/routes/captcha/[id]/result/GET.ts | 53 +++---- .../backend/routes/captcha/difficulty/GET.ts | 43 +++++ .../backend/routes/captcha/session/POST.ts | 3 +- packages/backend/routes/user/register.ts | 4 +- packages/backend/routes/video/[id]/info.ts | 4 +- packages/backend/src/routing.ts | 3 + packages/backend/src/schema.d.ts | 3 +- packages/backend/tsconfig.json | 1 + packages/core/db/dbNew.ts | 8 +- packages/core/db/pgConfigNew.ts | 8 - .../global.d.ts => core/db/psql.d.ts} | 2 +- packages/core/lib/randomID.ts | 149 ++---------------- packages/core/mq/rateLimiter.ts | 6 +- packages/core/mq/slidingWindow.ts | 57 +++++++ packages/core/mq/tokenBucket.ts | 44 +++++- packages/core/test/lib/randomID.test.ts | 64 +------- packages/crawler/db/bilibili_metadata.ts | 2 +- packages/crawler/db/snapshot.ts | 2 +- packages/crawler/db/snapshotSchedule.ts | 2 +- packages/crawler/db/songs.ts | 2 +- packages/crawler/mq/scheduling.ts | 2 +- packages/crawler/mq/task/collectSongs.ts | 2 +- packages/crawler/mq/task/getVideoDetails.ts | 2 +- packages/crawler/mq/task/getVideoStats.ts | 2 +- packages/crawler/mq/task/queueLatestVideo.ts | 2 +- .../mq/task/regularSnapshotInterval.ts | 2 +- 37 files changed, 408 insertions(+), 354 deletions(-) delete mode 100644 deno.json delete mode 100644 packages/backend/db/config.ts delete mode 100644 packages/backend/db/db.ts create mode 100644 packages/backend/lib/auth/captchaDifficulty.ts create mode 100644 packages/backend/lib/auth/getJWTsecret.ts create mode 100644 packages/backend/middleware/captcha.ts create mode 100644 packages/backend/routes/captcha/difficulty/GET.ts rename packages/{crawler/global.d.ts => core/db/psql.d.ts} (51%) create mode 100644 packages/core/mq/slidingWindow.ts diff --git a/.idea/cvsa.iml b/.idea/cvsa.iml index 7bfcf20..916ca6a 100644 --- a/.idea/cvsa.iml +++ b/.idea/cvsa.iml @@ -28,6 +28,8 @@ + + diff --git a/deno.json b/deno.json deleted file mode 100644 index 20c75ba..0000000 --- a/deno.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "lock": false, - "workspace": ["./packages/crawler", "./packages/core"], - "nodeModulesDir": "auto", - "tasks": { - "crawler": "deno task --filter 'crawler' all", - "backend": "deno task --filter 'backend' start" - }, - "fmt": { - "useTabs": true, - "lineWidth": 120, - "indentWidth": 4, - "semiColons": true, - "proseWrap": "always" - } -} diff --git a/packages/backend/db/config.ts b/packages/backend/db/config.ts deleted file mode 100644 index 9d0e514..0000000 --- a/packages/backend/db/config.ts +++ /dev/null @@ -1,46 +0,0 @@ -const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT", "DB_NAME_CRED"]; - -const unsetVars = requiredEnvVars.filter((key) => process.env[key] === undefined); - -if (unsetVars.length > 0) { - throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`); -} - -const databaseHost = process.env["DB_HOST"]!; -const databaseName = process.env["DB_NAME"]; -const databaseNameCred = process.env["DB_NAME_CRED"]!; -const databaseUser = process.env["DB_USER"]!; -const databasePassword = process.env["DB_PASSWORD"]!; -const databasePort = process.env["DB_PORT"]!; - -export const postgresConfig = { - hostname: databaseHost, - port: parseInt(databasePort), - database: databaseName, - user: databaseUser, - password: databasePassword -}; - -export const postgresConfigNpm = { - host: databaseHost, - port: parseInt(databasePort), - database: databaseName, - username: databaseUser, - password: databasePassword -}; - -export const postgresCredConfigNpm = { - host: databaseHost, - port: parseInt(databasePort), - database: databaseNameCred, - username: databaseUser, - password: databasePassword -}; - -export const postgresConfigCred = { - hostname: databaseHost, - port: parseInt(databasePort), - database: databaseNameCred, - user: databaseUser, - password: databasePassword -}; diff --git a/packages/backend/db/db.ts b/packages/backend/db/db.ts deleted file mode 100644 index 6dd488b..0000000 --- a/packages/backend/db/db.ts +++ /dev/null @@ -1,5 +0,0 @@ -import postgres from "postgres"; -import { postgresConfigNpm, postgresCredConfigNpm } from "./config"; - -export const sql = postgres(postgresConfigNpm); -export const sqlCred = postgres(postgresCredConfigNpm); diff --git a/packages/backend/db/snapshots.ts b/packages/backend/db/snapshots.ts index d5b8fa5..83ea7b5 100644 --- a/packages/backend/db/snapshots.ts +++ b/packages/backend/db/snapshots.ts @@ -1,4 +1,4 @@ -import { sql } from "./db"; +import { sql } from "@core/db/dbNew"; import type { VideoSnapshotType } from "@core/db/schema.d.ts"; export async function getVideoSnapshots( diff --git a/packages/backend/lib/auth/captchaDifficulty.ts b/packages/backend/lib/auth/captchaDifficulty.ts new file mode 100644 index 0000000..582fae7 --- /dev/null +++ b/packages/backend/lib/auth/captchaDifficulty.ts @@ -0,0 +1,57 @@ +import { Psql } from "@core/db/psql"; +import { SlidingWindow } from "@core/mq/slidingWindow.ts"; +import { redis } from "@core/db/redis.ts"; + +type seconds = number; + +export interface CaptchaDifficultyConfig { + global: boolean; + duration: seconds; + threshold: number; + difficulty: number; +} + +export const getCaptchaDifficultyConfigByRoute = async (sql: Psql, route: string): Promise => { + return sql` + SELECT duration, threshold, difficulty, global + FROM captcha_difficulty_settings + WHERE CONCAT(method, '-', path) = ${route} + ORDER BY duration + `; +}; + +export const getCaptchaConfigMaxDuration = async (sql: Psql, route: string): Promise => { + const rows = await sql<{max: number}[]>` + SELECT MAX(duration) + FROM captcha_difficulty_settings + WHERE CONCAT(method, '-', path) = ${route} + `; + if (rows.length < 1){ + return Number.MAX_SAFE_INTEGER; + } + return rows[0].max; +} + + +export const getCurrentCaptchaDifficulty = async (sql: Psql, route: string): Promise => { + const configs = await getCaptchaDifficultyConfigByRoute(sql, route); + if (configs.length < 1) { + return null + } + else if (configs.length == 1) { + return configs[0].difficulty + } + const maxDuration = configs.reduce((max, config) => + Math.max(max, config.duration), 0); + const slidingWindow = new SlidingWindow(redis, maxDuration); + for (let i = 0; i < configs.length; i++) { + const config = configs[i]; + const lastConfig = configs[i - 1]; + const count = await slidingWindow.count(`captcha-${route}`, config.duration); + if (count >= config.threshold) { + continue; + } + return lastConfig.difficulty + } + return configs[0].difficulty; +} diff --git a/packages/backend/lib/auth/getJWTsecret.ts b/packages/backend/lib/auth/getJWTsecret.ts new file mode 100644 index 0000000..9388892 --- /dev/null +++ b/packages/backend/lib/auth/getJWTsecret.ts @@ -0,0 +1,13 @@ +import { ErrorResponse } from "src/schema"; + +export const getJWTsecret = () => { + const secret = process.env["JWT_SECRET"]; + if (!secret) { + const response: ErrorResponse = { + message: "JWT_SECRET is not set", + code: "SERVER_ERROR" + }; + return [response, true]; + } + return [secret, null]; +} \ No newline at end of file diff --git a/packages/backend/middleware/captcha.ts b/packages/backend/middleware/captcha.ts new file mode 100644 index 0000000..ceef077 --- /dev/null +++ b/packages/backend/middleware/captcha.ts @@ -0,0 +1,112 @@ +import { Context, Next } from "hono"; +import { ErrorResponse } from "src/schema"; +import { SlidingWindow } from "@core/mq/slidingWindow.ts"; +import { getCaptchaConfigMaxDuration, getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; +import { sqlCred } from "@core/db/dbNew.ts"; +import { redis } from "@core/db/redis.ts"; +import { verify } from 'hono/jwt'; +import { JwtTokenInvalid, JwtTokenExpired } from "hono/utils/jwt/types"; +import { getJWTsecret } from "@/lib/auth/getJWTsecret.ts"; +import { lockManager } from "@core/mq/lockManager.ts"; +import { object, string, number, ValidationError } from "yup"; + +const tokenSchema = object({ + exp: number().integer(), + id: string().length(6), + difficulty: number().integer().moreThan(0) +}); + +export const captchaMiddleware = async (c: Context, next: Next) => { + const authHeader = c.req.header("Authorization"); + + if (!authHeader) { + const response: ErrorResponse = { + message: "'Authorization' header is missing.", + code: "UNAUTHORIZED" + }; + return c.json(response, 401); + } + + const authIsBearer = authHeader.startsWith("Bearer "); + if (!authIsBearer || authHeader.length < 8) { + const response: ErrorResponse = { + message: "'Authorization' header is invalid.", + code: "INVALID_HEADER" + }; + return c.json(response, 400); + } + + const [r, err] = getJWTsecret(); + if (err) { + return c.json(r as ErrorResponse, 500); + } + const jwtSecret = r as string; + + const token = authHeader.substring(7); + + const path = c.req.path; + const method = c.req.method; + const route = `${method}-${path}`; + + const requiredDifficulty = await getCurrentCaptchaDifficulty(sqlCred, route); + + try { + const decodedPayload = await verify(token, jwtSecret); + const payload = await tokenSchema.validate(decodedPayload); + const difficulty = payload.difficulty; + const tokenID = payload.id; + const consumed = await lockManager.isLocked(tokenID); + if (consumed) { + const response: ErrorResponse = { + message: "Token has already been used.", + code: "INVALID_CREDENTIALS" + }; + return c.json(response, 401); + } + if (difficulty < requiredDifficulty) { + const response: ErrorResponse = { + message: "Token to weak.", + code: "UNAUTHORIZED" + }; + return c.json(response, 401); + } + const EXPIRE_FIVE_MINUTES = 300; + await lockManager.acquireLock(tokenID, EXPIRE_FIVE_MINUTES); + } + catch (e) { + if (e instanceof JwtTokenInvalid) { + const response: ErrorResponse = { + message: "Failed to verify the token.", + code: "INVALID_CREDENTIALS" + }; + return c.json(response, 400); + } + else if (e instanceof JwtTokenExpired) { + const response: ErrorResponse = { + message: "Token expired.", + code: "INVALID_CREDENTIALS" + }; + return c.json(response, 400); + } + else if (e instanceof ValidationError) { + const response: ErrorResponse = { + code: "INVALID_QUERY_PARAMS", + message: "Invalid query parameters", + errors: e.errors + }; + return c.json(response, 400); + } + else { + const response: ErrorResponse = { + message: "Unknown error.", + code: "UNKNOWN_ERROR" + }; + return c.json(response, 500); + } + } + const duration = await getCaptchaConfigMaxDuration(sqlCred, route); + const window = new SlidingWindow(redis, duration); + await window.event(`captcha-${route}`); + + await next(); +}; \ No newline at end of file diff --git a/packages/backend/middleware/index.ts b/packages/backend/middleware/index.ts index 932daf0..6f8e411 100644 --- a/packages/backend/middleware/index.ts +++ b/packages/backend/middleware/index.ts @@ -7,6 +7,7 @@ import { preetifyResponse } from "./preetifyResponse.ts"; import { logger } from "./logger.ts"; import { timing } from "hono/timing"; import { contentType } from "./contentType.ts"; +import { captchaMiddleware } from "./captcha.ts"; export function configureMiddleWares(app: Hono<{ Variables: Variables }>) { app.use("*", contentType); @@ -15,5 +16,6 @@ export function configureMiddleWares(app: Hono<{ Variables: Variables }>) { app.use("*", logger({})); app.post("/user", registerRateLimiter); + app.post("/user", captchaMiddleware); app.all("/ping", bodyLimitForPing, ...pingHandler); } diff --git a/packages/backend/middleware/rateLimiters.ts b/packages/backend/middleware/rateLimiters.ts index a065f4c..e937406 100644 --- a/packages/backend/middleware/rateLimiters.ts +++ b/packages/backend/middleware/rateLimiters.ts @@ -5,27 +5,31 @@ import { getConnInfo } from "hono/bun"; import type { Context } from "hono"; import { redis } from "@core/db/redis.ts"; import { RedisStore } from "rate-limit-redis"; +import { generateRandomId } from "@core/lib/randomID.ts"; + +export const getIdentifier = (c: Context, includeIP: boolean = true) => { + let ipAddr = generateRandomId(6); + const info = getConnInfo(c); + if (info.remote && info.remote.address) { + ipAddr = info.remote.address; + } + const forwardedFor = c.req.header("X-Forwarded-For"); + if (forwardedFor) { + ipAddr = forwardedFor.split(",")[0]; + } + const path = c.req.path; + const method = c.req.method; + const ipIdentifier = includeIP ? `@${ipAddr}` : ""; + return `${method}-${path}${ipIdentifier}` +} export const registerRateLimiter = rateLimiter({ windowMs: 60 * MINUTE, limit: 10, standardHeaders: "draft-6", - keyGenerator: (c) => { - let ipAddr = crypto.randomUUID() as string; - const info = getConnInfo(c as unknown as Context); - if (info.remote && info.remote.address) { - ipAddr = info.remote.address; - } - const forwardedFor = c.req.header("X-Forwarded-For"); - if (forwardedFor) { - ipAddr = forwardedFor.split(",")[0]; - } - const path = new URL(c.req.url).pathname; - const method = c.req.method; - return `${method}-${path}@${ipAddr}`; - }, + keyGenerator: getIdentifier, store: new RedisStore({ // @ts-expect-error - Known issue: the `c`all` function is not present in @types/ioredis sendCommand: (...args: string[]) => redis.call(...args) }) as unknown as Store -}); +}); \ No newline at end of file diff --git a/packages/backend/package.json b/packages/backend/package.json index aede4e7..2cc883d 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -10,7 +10,6 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", - "jose": "^6.0.11", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts index cffab6d..25d307c 100644 --- a/packages/backend/routes/captcha/[id]/result/GET.ts +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -2,9 +2,9 @@ import { Context } from "hono"; import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; import { createHandlers } from "src/utils.ts"; -import * as jose from "jose"; +import { sign } from 'hono/jwt' import { generateRandomId } from "@core/lib/randomID.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; +import { getJWTsecret } from "lib/auth/getJWTsecret.ts"; interface CaptchaResponse { success: boolean; @@ -32,60 +32,55 @@ export const verifyChallengeHandler = createHandlers( const id = c.req.param("id"); const ans = c.req.query("ans"); if (!ans) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Missing required query parameter: ans", code: "INVALID_QUERY_PARAMS" }; - return c.json>(response, 400); + return c.json(response, 400); } const res = await getChallengeVerificationResult(id, ans); const data: CaptchaResponse = await res.json(); if (data.error && res.status === 404) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: data.error, code: "ENTITY_NOT_FOUND" }; - return c.json>(response, 401); + return c.json(response, 401); } else if (data.error && res.status === 400) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: data.error, code: "INVALID_QUERY_PARAMS" }; - return c.json>(response, 400); + return c.json(response, 400); } else if (data.error) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: data.error, code: "UNKNOWN_ERROR" }; - return c.json>(response, 500); + return c.json(response, 500); } if (!data.success) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Incorrect answer", code: "INVALID_CREDENTIALS" }; - return c.json>(response, 401); + return c.json(response, 401); } - const secret = process.env["JWT_SECRET"]; - if (!secret) { - const response: ErrorResponse = { - message: "JWT_SECRET is not set", - code: "SERVER_ERROR" - }; - return c.json>(response, 500); + const [r, err] = getJWTsecret(); + if (err) { + return c.json(r as ErrorResponse, 500); } - const jwtSecret = new TextEncoder().encode(secret); - const alg = "HS256"; + const jwtSecret = r as string; - - const tokenID = generateRandomId(10); - const EXPIRE_FIVE_MINUTES = 300; - await lockManager.acquireLock(tokenID, EXPIRE_FIVE_MINUTES); - const jwt = await new jose.SignJWT({ difficulty: data.difficulty!, id: tokenID }) - .setProtectedHeader({ alg }) - .setIssuedAt() - .sign(jwtSecret); + const tokenID = generateRandomId(6); + const NOW = Math.floor(Date.now() / 1000) + const FIVE_MINUTES_LATER = NOW + 60 * 5; + const jwt = await sign({ + difficulty: data.difficulty!, + id: tokenID, + exp: FIVE_MINUTES_LATER + }, jwtSecret); return c.json({ token: jwt }); diff --git a/packages/backend/routes/captcha/difficulty/GET.ts b/packages/backend/routes/captcha/difficulty/GET.ts new file mode 100644 index 0000000..0bed7cc --- /dev/null +++ b/packages/backend/routes/captcha/difficulty/GET.ts @@ -0,0 +1,43 @@ +import { createHandlers } from "src/utils.ts"; +import { object, string, ValidationError } from "yup"; +import { ErrorResponse } from "src/schema"; +import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; +import { sqlCred } from "@core/db/dbNew.ts"; + +const queryParamsSchema = object({ + route: string().matches(/(?:GET|POST|PUT|PATCH|DELETE)-\/.*/g) +}); + +export const getCaptchaDifficultyHandler = createHandlers(async (c) => { + try { + const queryParams = await queryParamsSchema.validate(c.req.query()); + const { route } = queryParams; + const difficulty = await getCurrentCaptchaDifficulty(sqlCred, route); + if (!difficulty) { + const response: ErrorResponse = { + code: "ENTITY_NOT_FOUND", + message: "No difficulty configs found for this route." + }; + return c.json>(response, 404); + } + return c.json({ + "difficulty": difficulty + }); + } catch (e: unknown) { + if (e instanceof ValidationError) { + const response: ErrorResponse = { + code: "INVALID_QUERY_PARAMS", + message: "Invalid query parameters", + errors: e.errors + }; + return c.json(response, 400); + } else { + const response: ErrorResponse = { + code: "UNKNOWN_ERROR", + message: "Unknown error", + errors: [e] + }; + return c.json>(response, 500); + } + } +}); diff --git a/packages/backend/routes/captcha/session/POST.ts b/packages/backend/routes/captcha/session/POST.ts index 63b9331..7d2c3d1 100644 --- a/packages/backend/routes/captcha/session/POST.ts +++ b/packages/backend/routes/captcha/session/POST.ts @@ -6,7 +6,7 @@ const createNewChallenge = async (difficulty: number) => { const baseURL = process.env["UCAPTCHA_URL"]; const url = new URL(baseURL); url.pathname = "/challenge"; - const res = await fetch(url.toString(), { + return await fetch(url.toString(), { method: "POST", headers: { "Content-Type": "application/json", @@ -15,7 +15,6 @@ const createNewChallenge = async (difficulty: number) => { difficulty: difficulty, }) }); - return res; } export const createCaptchaSessionHandler = createHandlers(async (_c) => { diff --git a/packages/backend/routes/user/register.ts b/packages/backend/routes/user/register.ts index ac8200d..d774b56 100644 --- a/packages/backend/routes/user/register.ts +++ b/packages/backend/routes/user/register.ts @@ -3,7 +3,7 @@ import Argon2id from "@rabbit-company/argon2id"; import { object, string, ValidationError } from "yup"; import type { Context } from "hono"; import type { Bindings, BlankEnv, BlankInput } from "hono/types"; -import { sqlCred } from "db/db.ts"; +import { sqlCred } from "@core/db/dbNew.ts"; import { ErrorResponse, StatusResponse } from "src/schema"; const RegistrationBodySchema = object({ @@ -64,7 +64,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => { return c.json>(response, 400); } else { const response: ErrorResponse = { - message: "Invalid JSON payload.", + message: "Unknown error.", errors: [(e as Error).message], code: "UNKNOWN_ERROR" }; diff --git a/packages/backend/routes/video/[id]/info.ts b/packages/backend/routes/video/[id]/info.ts index ad64045..3e05493 100644 --- a/packages/backend/routes/video/[id]/info.ts +++ b/packages/backend/routes/video/[id]/info.ts @@ -1,8 +1,8 @@ import logger from "@core/log/logger.ts"; import { redis } from "@core/db/redis.ts"; -import { sql } from "../../../db/db.ts"; +import { sql } from "@core/db/dbNew.ts"; import { number, ValidationError } from "yup"; -import { createHandlers } from "../../../src/utils.ts"; +import { createHandlers } from "@/src/utils.ts"; import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo.ts"; import { idSchema } from "./snapshots.ts"; import { NetSchedulerError } from "@core/net/delegate.ts"; diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts index a56ed73..d5a7aa2 100644 --- a/packages/backend/src/routing.ts +++ b/packages/backend/src/routing.ts @@ -5,6 +5,7 @@ import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; import { Hono } from "hono"; import { Variables } from "hono/types"; import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha"; +import { getCaptchaDifficultyHandler } from "../routes/captcha/difficulty/GET.ts"; export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.get("/", ...rootHandler); @@ -17,4 +18,6 @@ export function configureRoutes(app: Hono<{ Variables: Variables }>) { app.post("/captcha/session", ...createCaptchaSessionHandler); app.get("/captcha/:id/result", ...verifyChallengeHandler); + + app.get("/captcha/difficulty", ...getCaptchaDifficultyHandler) } diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index 51f54fa..4a50d7b 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -3,13 +3,14 @@ type ErrorCode = | "UNKNOWN_ERROR" | "INVALID_PAYLOAD" | "INVALID_FORMAT" + | "INVALID_HEADER" | "BODY_TOO_LARGE" | "UNAUTHORIZED" | "INVALID_CREDENTIALS" | "ENTITY_NOT_FOUND" | "SERVER_ERROR"; -export interface ErrorResponse { +export interface ErrorResponse { code: ErrorCode; message: string; errors?: E[]; diff --git a/packages/backend/tsconfig.json b/packages/backend/tsconfig.json index 6cbd96f..6b2128e 100644 --- a/packages/backend/tsconfig.json +++ b/packages/backend/tsconfig.json @@ -10,6 +10,7 @@ "skipLibCheck": true, "paths": { "@core/*": ["../core/*"], + "@/*": ["./*"], "@crawler/*": ["../crawler/*"] }, "allowSyntheticDefaultImports": true, diff --git a/packages/core/db/dbNew.ts b/packages/core/db/dbNew.ts index 11088e5..2f925ec 100644 --- a/packages/core/db/dbNew.ts +++ b/packages/core/db/dbNew.ts @@ -1,6 +1,8 @@ import postgres from "postgres"; -import { postgresConfigNpm } from "./pgConfigNew"; +import { postgresConfigCred, postgresConfig } from "./pgConfigNew"; -export const sql = postgres(postgresConfigNpm); +export const sql = postgres(postgresConfig); -export const sqlTest = postgres(postgresConfigNpm); \ No newline at end of file +export const sqlCred = postgres(postgresConfigCred); + +export const sqlTest = postgres(postgresConfig); \ No newline at end of file diff --git a/packages/core/db/pgConfigNew.ts b/packages/core/db/pgConfigNew.ts index d6cb437..e41c02f 100644 --- a/packages/core/db/pgConfigNew.ts +++ b/packages/core/db/pgConfigNew.ts @@ -14,14 +14,6 @@ const databasePassword = process.env["DB_PASSWORD"]!; const databasePort = process.env["DB_PORT"]!; export const postgresConfig = { - hostname: databaseHost, - port: parseInt(databasePort), - database: databaseName, - user: databaseUser, - password: databasePassword -}; - -export const postgresConfigNpm = { host: databaseHost, port: parseInt(databasePort), database: databaseName, diff --git a/packages/crawler/global.d.ts b/packages/core/db/psql.d.ts similarity index 51% rename from packages/crawler/global.d.ts rename to packages/core/db/psql.d.ts index 37fc0e2..f629cd5 100644 --- a/packages/crawler/global.d.ts +++ b/packages/core/db/psql.d.ts @@ -1,3 +1,3 @@ import type postgres from "postgres"; -export type Psql = postgres.Sql<{}>; +export type Psql = postgres.Sql; diff --git a/packages/core/lib/randomID.ts b/packages/core/lib/randomID.ts index 477fe4a..7448438 100644 --- a/packages/core/lib/randomID.ts +++ b/packages/core/lib/randomID.ts @@ -1,142 +1,15 @@ -const getSecureRandomInt = (max: number): number => { - const array = new Uint32Array(1); - crypto.getRandomValues(array); - // Using modulo bias is technically present, but negligible here because the space (56) is tiny compared to 2^32. - return array[0] % max; -} - - -/** - * Generates a random ID with characteristics similar to UUIDv7, - * incorporating a timestamp prefix for sortability and a random suffix, - * using a customizable length and a specific character set. - * - * This function aims for sortability by placing a time-based component at the beginning, - * similar to UUIDv7, while allowing a variable total length and using a character set - * designed to avoid visually similar characters. - * - * The character set includes uppercase and lowercase letters and numbers, - * excluding visually similar characters (0, O, I, l, 1). - * - * **Length Reference**: - * - * With a collision probability of **0.1%**, - * the maximum ID generation rate per millisecond for the following lengths is: - * - **10**: 1.8 IDs / ms or 1,844 QPS - * - **12**: 27 IDs / ms or 26,998 QPS - * - **16**: 5784 IDs / ms or 5,784,295 QPS - * - * With a collision probability of **0.001%**, - * the maximum ID generation rate per millisecond for the following lengths is: - * - **11**: 1.5 IDs / ms or 1,520 QPS - * - **14**: 85 IDs / ms or 85,124 QPS - * - **16**: 1246 IDs / ms or 1,245,983 QPS - * - * With a collision probability of **0.00001%**, - * the maximum ID generation rate per millisecond for the following lengths is: - * - **14**: 18 IDs / ms or 18,339 QPS - * - **15**: 70 IDs / ms or 70,164 QPS - * - **16**: 1246 IDs / ms or 268,438 QPS - * - * The formula: max_qps = 1000 * (2 * (56**(length - 8)) * -log(1 - prob))**(1/3) - * - * @param length The desired total length of the ID. Must be at least 8. - * @returns A sortable random ID string of the specified length. - * @throws Error if the requested length is less than the minimum required for the timestamp prefix (8). - */ export function generateRandomId(length: number): string { - // Character set excluding 0, O, I, l, 1 - const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters - const base = allowedChars.length; // 56 - const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part to ensure sortability + const characters = 'abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789'; + const charactersLength = characters.length; + const randomBytes = new Uint8Array(length); - if (length < TIMESTAMP_PREFIX_LENGTH) { - throw new Error(`Length must be at least ${TIMESTAMP_PREFIX_LENGTH} to include the timestamp prefix.`); + crypto.getRandomValues(randomBytes); + + let result = ''; + for (let i = 0; i < length; i++) { + const randomIndex = randomBytes[i] % charactersLength; + result += characters.charAt(randomIndex); } - // --- Generate Timestamp Prefix --- - const timestamp = Date.now(); // Milliseconds since epoch (Unix time) - let timestampBaseString = ""; - let tempTimestamp = timestamp; - const firstChar = allowedChars[0]; // Character for padding ('a') - - // Convert timestamp to a base-56 string - // We process from the least significant "digit" to the most significant - while (tempTimestamp > 0) { - timestampBaseString = allowedChars[tempTimestamp % base] + timestampBaseString; - tempTimestamp = Math.floor(tempTimestamp / base); - } - - // Pad the timestamp string at the beginning to ensure a fixed length. - // This is crucial for chronological sortability of the generated IDs. - while (timestampBaseString.length < TIMESTAMP_PREFIX_LENGTH) { - timestampBaseString = firstChar + timestampBaseString; - } - - // Although highly unlikely with an 8-character prefix using base 56 for current timestamps, - // this would truncate if the timestamp string somehow exceeded the prefix length. - if (timestampBaseString.length > TIMESTAMP_PREFIX_LENGTH) { - timestampBaseString = timestampBaseString.substring(timestampBaseString.length - TIMESTAMP_PREFIX_LENGTH); - } - - // --- Generate Random Suffix --- - const randomLength = length - TIMESTAMP_PREFIX_LENGTH; - let randomSuffix = ""; - const allowedCharsLength = allowedChars.length; - - for (let i = 0; i < randomLength; i++) { - const randomIndex = getSecureRandomInt(allowedCharsLength); - randomSuffix += allowedChars[randomIndex]; - } - - // --- Concatenate and Return --- - return timestampBaseString + randomSuffix; -} - -/** - * Decodes the timestamp (in milliseconds since epoch) from an ID - * generated by a function that uses a fixed-length timestamp prefix - * encoded in a specific base and character set (like the previous example). - * - * It extracts the timestamp prefix and converts it back from the - * custom base-56 encoding to a number. - * - * @param id The ID string containing the timestamp prefix. - * @returns The timestamp in milliseconds since epoch. - * @throws Error if the ID is too short or contains invalid characters in the timestamp prefix. - */ -export function decodeTimestampFromId(id: string): number { - // Character set must match the encoding function used to generate the ID - const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; // 56 characters - const base = allowedChars.length; // 56 - const TIMESTAMP_PREFIX_LENGTH = 8; // Fixed length for the timestamp part - - if (id.length < TIMESTAMP_PREFIX_LENGTH) { - throw new Error( - `ID must be at least ${TIMESTAMP_PREFIX_LENGTH} characters long to contain a timestamp prefix.` - ); - } - - // Extract the timestamp prefix from the beginning of the ID string - const timestampPrefix = id.substring(0, TIMESTAMP_PREFIX_LENGTH); - let timestamp = 0; - - // Convert the base-56 timestamp string back to a number - // Iterate through the prefix characters from left to right (most significant 'digit') - for (let i = 0; i < timestampPrefix.length; i++) { - const char = timestampPrefix[i]; - // Find the index (value) of the character in the allowed character set - const charIndex = allowedChars.indexOf(char); - - if (charIndex === -1) { - // If a character is not in the allowed set, the ID is likely invalid - throw new Error(`Invalid character "${char}" found in timestamp prefix.`); - } - - // Standard base conversion: accumulate the value - // For each digit, multiply the current total by the base and add the digit's value - timestamp = timestamp * base + charIndex; - } - - return timestamp; -} + return result; +} \ No newline at end of file diff --git a/packages/core/mq/rateLimiter.ts b/packages/core/mq/rateLimiter.ts index 96f208c..d15ddd5 100644 --- a/packages/core/mq/rateLimiter.ts +++ b/packages/core/mq/rateLimiter.ts @@ -8,7 +8,7 @@ export interface RateLimiterConfig { export class RateLimiter { private configs: RateLimiterConfig[] = []; private buckets: TokenBucket[] = []; - private identifierFn: (configIndex: number) => string; + private readonly identifierFn: (configIndex: number) => string; /* * @param name The name of the rate limiter @@ -26,8 +26,8 @@ export class RateLimiter { for (let i = 0; i < configs.length; i++) { const config = configs[i]; const bucket = new TokenBucket({ - capacity: config.max, - rate: config.max / config.duration, + max: config.max, + duration: config.duration, identifier: this.identifierFn(i), }) this.buckets.push(bucket); diff --git a/packages/core/mq/slidingWindow.ts b/packages/core/mq/slidingWindow.ts new file mode 100644 index 0000000..092190c --- /dev/null +++ b/packages/core/mq/slidingWindow.ts @@ -0,0 +1,57 @@ +import type { Redis } from "ioredis"; + +export class SlidingWindow { + private redis: Redis; + private readonly windowSize: number; + + /* + * Create a new sliding window + * @param redisClient The Redis client used to store the data + * @param windowSize The size of the window in seconds + */ + constructor(redisClient: Redis, windowSize: number) { + this.redis = redisClient; + this.windowSize = windowSize * 1000; + } + + /* + * Trigger an event in the sliding window + * @param eventName The name of the event + */ + async event(eventName: string): Promise { + const now = Date.now(); + const key = `cvsa:sliding_window:${eventName}`; + + const uniqueMember = `${now}-${Math.random()}`; + // Add current timestamp to an ordered set + await this.redis.zadd(key, now, uniqueMember); + + // Remove timestamps outside the window + await this.redis.zremrangebyscore(key, 0, now - this.windowSize); + } + + /* + * Count the number of events in the sliding window + * @param {string} eventName The name of the event + * @param {number} [duration] The duration of the window in seconds + */ + async count(eventName: string, duration?: number): Promise { + const key = `cvsa:sliding_window:${eventName}`; + const now = Date.now(); + + // Remove timestamps outside the window + await this.redis.zremrangebyscore(key, 0, now - this.windowSize); + + if (duration) { + return this.redis.zcount(key, now - duration * 1000, now); + } + + // Get the number of timestamps in the window + return this.redis.zcard(key); + } + + clear(eventName: string): Promise { + const key = `cvsa:sliding_window:${eventName}`; + return this.redis.del(key); + } +} \ No newline at end of file diff --git a/packages/core/mq/tokenBucket.ts b/packages/core/mq/tokenBucket.ts index 8e94db9..e56d368 100644 --- a/packages/core/mq/tokenBucket.ts +++ b/packages/core/mq/tokenBucket.ts @@ -1,28 +1,56 @@ import { redis } from "@core/db/redis"; import { SECOND } from "@core/const/time"; -interface TokenBucketOptions { +export interface TokenBucketRateOptions { capacity: number; rate: number; identifier: string; - keyPrefix?: string; + keyPrefix?: string; } +export interface TokenBucketDurationOptions { + duration: number; + max: number; + identifier: string; + keyPrefix?: string; +} + +export type TokenBucketConstructorOptions = TokenBucketRateOptions | TokenBucketDurationOptions; + export class TokenBucket { private readonly capacity: number; private readonly rate: number; private readonly keyPrefix: string; private readonly identifier: string; - constructor(options: TokenBucketOptions) { - if (options.capacity <= 0 || options.rate <= 0) { - throw new Error("Capacity and rate must be greater than zero."); + constructor(options: TokenBucketConstructorOptions) { + if (!options.identifier) { + throw new Error("Identifier is required."); } - - this.capacity = options.capacity; - this.rate = options.rate; this.identifier = options.identifier; this.keyPrefix = options.keyPrefix || "cvsa:token_bucket:"; + + const isRateOptions = 'capacity' in options && 'rate' in options; + const isDurationOptions = 'duration' in options && 'max' in options; + + if (isRateOptions && isDurationOptions) { + throw new Error("Provide either 'capacity'/'rate' or 'duration'/'max', not both."); + } else if (isRateOptions) { + if (options.capacity <= 0 || options.rate <= 0) { + throw new Error("'capacity' and 'rate' must be greater than zero."); + } + this.capacity = options.capacity; + this.rate = options.rate; + } else if (isDurationOptions) { + if (options.duration <= 0 || options.max <= 0) { + throw new Error("'duration' and 'max' must be greater than zero."); + } + this.capacity = options.max; + this.rate = options.max / options.duration; + + } else { + throw new Error("Provide either 'capacity'/'rate' or 'duration'/'max'."); + } } getKey(): string { diff --git a/packages/core/test/lib/randomID.test.ts b/packages/core/test/lib/randomID.test.ts index 71b859c..8f69dcd 100644 --- a/packages/core/test/lib/randomID.test.ts +++ b/packages/core/test/lib/randomID.test.ts @@ -1,24 +1,13 @@ import { describe, expect, it } from "vitest"; -import { generateRandomId, decodeTimestampFromId } from "@core/lib/randomID.ts"; +import { generateRandomId } from "@core/lib/randomID.ts"; describe("generateRandomId", () => { - it("should throw an error if the requested length is less than 8", () => { - expect(() => generateRandomId(7)).toThrowError("Length must be at least 8 to include the timestamp prefix."); - }); - it("should generate an ID of the specified length", () => { const length = 15; const id = generateRandomId(length); expect(id).toHaveLength(length); }); - it("should generate an ID with a timestamp prefix of length 8", () => { - const id = generateRandomId(12); - expect(id).toHaveProperty("substring"); - expect(id).toHaveProperty("length"); - expect(id.length).toBeGreaterThanOrEqual(8); - }); - it("should generate an ID containing only allowed characters", () => { const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; const id = generateRandomId(20); @@ -26,55 +15,4 @@ describe("generateRandomId", () => { expect(allowedChars).toContain(char); } }); - - it("should generate IDs that are sortable by creation time", () => { - const id1 = generateRandomId(10); - // Simulate a slight delay to ensure different timestamps - return new Promise((resolve) => { - setTimeout(() => { - const id2 = generateRandomId(10); - expect(id2 >= id1).toBe(true); - resolve(null); - }, 10); - }); - }); -}); - -describe("decodeTimestampFromId", () => { - it("should throw an error if the ID length is less than 8", () => { - expect(() => decodeTimestampFromId("abcdefg")).toThrowError( - "ID must be at least 8 characters long to contain a timestamp prefix." - ); - }); - - it("should throw an error if the timestamp prefix contains invalid characters", () => { - const invalidId = "0bcdefghijk"; - expect(() => decodeTimestampFromId(invalidId)).toThrowError('Invalid character "0" found in timestamp prefix.'); - }); - - it("should correctly decode the timestamp from a generated ID", () => { - const now = Date.now(); - // Mock Date.now to control the timestamp for testing - const originalDateNow = Date.now; - global.Date.now = () => now; - const id = generateRandomId(16); - global.Date.now = originalDateNow; // Restore original Date.now - - const decodedTimestamp = decodeTimestampFromId(id); - // Allow a small margin for potential timing differences in test execution - expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); - expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); - }); - - it("should correctly decode the timestamp even with a longer ID", () => { - const now = Date.now(); - const originalDateNow = Date.now; - global.Date.now = () => now; - const id = generateRandomId(20); - global.Date.now = originalDateNow; - - const decodedTimestamp = decodeTimestampFromId(id); - expect(decodedTimestamp).toBeGreaterThanOrEqual(now - 1); - expect(decodedTimestamp).toBeLessThanOrEqual(now + 1); - }); }); diff --git a/packages/crawler/db/bilibili_metadata.ts b/packages/crawler/db/bilibili_metadata.ts index acc136c..90f7513 100644 --- a/packages/crawler/db/bilibili_metadata.ts +++ b/packages/crawler/db/bilibili_metadata.ts @@ -1,4 +1,4 @@ -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; import { AllDataType, BiliUserType } from "@core/db/schema"; import { AkariModelVersion } from "ml/const"; diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index b4635ee..e070283 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -1,6 +1,6 @@ import { LatestSnapshotType } from "@core/db/schema"; import { SnapshotNumber } from "mq/task/getVideoStats.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export async function getVideosNearMilestone(sql: Psql) { const queryResult = await sql` diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index 9937a2d..a0cb7ec 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -4,7 +4,7 @@ import { MINUTE } from "@core/const/time.ts"; import { redis } from "@core/db/redis.ts"; import { Redis } from "ioredis"; import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; const REDIS_KEY = "cvsa:snapshot_window_counts"; diff --git a/packages/crawler/db/songs.ts b/packages/crawler/db/songs.ts index 5f5070f..3fbf487 100644 --- a/packages/crawler/db/songs.ts +++ b/packages/crawler/db/songs.ts @@ -1,4 +1,4 @@ -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts"; export async function getNotCollectedSongs(sql: Psql) { diff --git a/packages/crawler/mq/scheduling.ts b/packages/crawler/mq/scheduling.ts index b84d73f..cf7427f 100644 --- a/packages/crawler/mq/scheduling.ts +++ b/packages/crawler/mq/scheduling.ts @@ -2,7 +2,7 @@ import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db import { truncate } from "utils/truncate.ts"; import { closetMilestone } from "./exec/snapshotTick.ts"; import { HOUR, MINUTE } from "@core/const/time.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); diff --git a/packages/crawler/mq/task/collectSongs.ts b/packages/crawler/mq/task/collectSongs.ts index 1d7634d..a2fef58 100644 --- a/packages/crawler/mq/task/collectSongs.ts +++ b/packages/crawler/mq/task/collectSongs.ts @@ -3,7 +3,7 @@ import { aidExistsInSongs, getNotCollectedSongs } from "db/songs.ts"; import logger from "@core/log/logger.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; import { MINUTE } from "@core/const/time.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export async function collectSongs() { const aids = await getNotCollectedSongs(sql); diff --git a/packages/crawler/mq/task/getVideoDetails.ts b/packages/crawler/mq/task/getVideoDetails.ts index 80a5867..0d76ebb 100644 --- a/packages/crawler/mq/task/getVideoDetails.ts +++ b/packages/crawler/mq/task/getVideoDetails.ts @@ -4,7 +4,7 @@ import logger from "@core/log/logger.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata.ts"; import { HOUR, SECOND } from "@core/const/time.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export async function insertVideoInfo(sql: Psql, aid: number) { const videoExists = await videoExistsInAllData(sql, aid); diff --git a/packages/crawler/mq/task/getVideoStats.ts b/packages/crawler/mq/task/getVideoStats.ts index 2a7d7ff..49c67a8 100644 --- a/packages/crawler/mq/task/getVideoStats.ts +++ b/packages/crawler/mq/task/getVideoStats.ts @@ -1,6 +1,6 @@ import { getVideoInfo } from "@core/net/getVideoInfo.ts"; import logger from "@core/log/logger.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export interface SnapshotNumber { time: number; diff --git a/packages/crawler/mq/task/queueLatestVideo.ts b/packages/crawler/mq/task/queueLatestVideo.ts index af824f7..49362a1 100644 --- a/packages/crawler/mq/task/queueLatestVideo.ts +++ b/packages/crawler/mq/task/queueLatestVideo.ts @@ -4,7 +4,7 @@ import { sleep } from "utils/sleep.ts"; import { SECOND } from "@core/const/time.ts"; import logger from "@core/log/logger.ts"; import { LatestVideosQueue } from "mq/index.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export async function queueLatestVideos( sql: Psql, diff --git a/packages/crawler/mq/task/regularSnapshotInterval.ts b/packages/crawler/mq/task/regularSnapshotInterval.ts index 11871d8..852d401 100644 --- a/packages/crawler/mq/task/regularSnapshotInterval.ts +++ b/packages/crawler/mq/task/regularSnapshotInterval.ts @@ -1,6 +1,6 @@ import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts"; import { HOUR } from "@core/const/time.ts"; -import type { Psql } from "global.d.ts"; +import type { Psql } from "@core/db/global.d.ts"; export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => { const now = Date.now(); From fa5ccce83ff4a4805ac27079502089f80a278fc3 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 18 May 2025 00:36:39 +0800 Subject: [PATCH 10/12] fix: incorrect import of type `Psql` --- packages/crawler/db/bilibili_metadata.ts | 2 +- packages/crawler/db/snapshot.ts | 2 +- packages/crawler/db/snapshotSchedule.ts | 2 +- packages/crawler/db/songs.ts | 2 +- packages/crawler/mq/task/collectSongs.ts | 2 +- packages/crawler/mq/task/getVideoDetails.ts | 2 +- packages/crawler/mq/task/getVideoStats.ts | 2 +- packages/crawler/mq/task/queueLatestVideo.ts | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/crawler/db/bilibili_metadata.ts b/packages/crawler/db/bilibili_metadata.ts index 90f7513..6c68516 100644 --- a/packages/crawler/db/bilibili_metadata.ts +++ b/packages/crawler/db/bilibili_metadata.ts @@ -1,4 +1,4 @@ -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; import { AllDataType, BiliUserType } from "@core/db/schema"; import { AkariModelVersion } from "ml/const"; diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index e070283..16df13c 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -1,6 +1,6 @@ import { LatestSnapshotType } from "@core/db/schema"; import { SnapshotNumber } from "mq/task/getVideoStats.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; export async function getVideosNearMilestone(sql: Psql) { const queryResult = await sql` diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index a0cb7ec..6fcfdc1 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -4,7 +4,7 @@ import { MINUTE } from "@core/const/time.ts"; import { redis } from "@core/db/redis.ts"; import { Redis } from "ioredis"; import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; const REDIS_KEY = "cvsa:snapshot_window_counts"; diff --git a/packages/crawler/db/songs.ts b/packages/crawler/db/songs.ts index 3fbf487..ebdd08c 100644 --- a/packages/crawler/db/songs.ts +++ b/packages/crawler/db/songs.ts @@ -1,4 +1,4 @@ -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts"; export async function getNotCollectedSongs(sql: Psql) { diff --git a/packages/crawler/mq/task/collectSongs.ts b/packages/crawler/mq/task/collectSongs.ts index a2fef58..dcf5472 100644 --- a/packages/crawler/mq/task/collectSongs.ts +++ b/packages/crawler/mq/task/collectSongs.ts @@ -3,7 +3,7 @@ import { aidExistsInSongs, getNotCollectedSongs } from "db/songs.ts"; import logger from "@core/log/logger.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; import { MINUTE } from "@core/const/time.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; export async function collectSongs() { const aids = await getNotCollectedSongs(sql); diff --git a/packages/crawler/mq/task/getVideoDetails.ts b/packages/crawler/mq/task/getVideoDetails.ts index 0d76ebb..1fc618a 100644 --- a/packages/crawler/mq/task/getVideoDetails.ts +++ b/packages/crawler/mq/task/getVideoDetails.ts @@ -4,7 +4,7 @@ import logger from "@core/log/logger.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata.ts"; import { HOUR, SECOND } from "@core/const/time.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; export async function insertVideoInfo(sql: Psql, aid: number) { const videoExists = await videoExistsInAllData(sql, aid); diff --git a/packages/crawler/mq/task/getVideoStats.ts b/packages/crawler/mq/task/getVideoStats.ts index 49c67a8..ffec09f 100644 --- a/packages/crawler/mq/task/getVideoStats.ts +++ b/packages/crawler/mq/task/getVideoStats.ts @@ -1,6 +1,6 @@ import { getVideoInfo } from "@core/net/getVideoInfo.ts"; import logger from "@core/log/logger.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; export interface SnapshotNumber { time: number; diff --git a/packages/crawler/mq/task/queueLatestVideo.ts b/packages/crawler/mq/task/queueLatestVideo.ts index 49362a1..9f583c2 100644 --- a/packages/crawler/mq/task/queueLatestVideo.ts +++ b/packages/crawler/mq/task/queueLatestVideo.ts @@ -4,7 +4,7 @@ import { sleep } from "utils/sleep.ts"; import { SECOND } from "@core/const/time.ts"; import logger from "@core/log/logger.ts"; import { LatestVideosQueue } from "mq/index.ts"; -import type { Psql } from "@core/db/global.d.ts"; +import type { Psql } from "@core/db/psql.d.ts"; export async function queueLatestVideos( sql: Psql, From c5ba67306929c8adec107c09c1c44f7dd5371a70 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 18 May 2025 03:28:27 +0800 Subject: [PATCH 11/12] fix: several bugs in captcha rate limiting --- .idea/compiler.xml | 6 - bun.lock | 136 +++++++++++++++--- .../backend/lib/auth/captchaDifficulty.ts | 13 +- packages/backend/middleware/captcha.ts | 11 +- packages/backend/middleware/rateLimiters.ts | 43 +++--- .../backend/routes/captcha/session/POST.ts | 44 +++++- packages/backend/src/schema.d.ts | 3 +- packages/core/mq/multipleRateLimiter.ts | 63 ++++++++ packages/core/mq/rateLimiter.ts | 68 --------- packages/core/mq/tokenBucket.ts | 112 --------------- packages/core/net/delegate.ts | 10 +- packages/core/package.json | 1 + 12 files changed, 273 insertions(+), 237 deletions(-) delete mode 100644 .idea/compiler.xml create mode 100644 packages/core/mq/multipleRateLimiter.ts delete mode 100644 packages/core/mq/rateLimiter.ts delete mode 100644 packages/core/mq/tokenBucket.ts diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 8ca546d..0000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - \ No newline at end of file diff --git a/bun.lock b/bun.lock index f5d6db7..251cf70 100644 --- a/bun.lock +++ b/bun.lock @@ -19,7 +19,6 @@ "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", - "jose": "^6.0.11", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", @@ -34,6 +33,7 @@ "packages/core": { "name": "core", "dependencies": { + "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", @@ -235,6 +235,8 @@ "@jridgewell/trace-mapping": ["@jridgewell/trace-mapping@0.3.25", "", { "dependencies": { "@jridgewell/resolve-uri": "^3.1.0", "@jridgewell/sourcemap-codec": "^1.4.14" } }, "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ=="], + "@koshnic/ratelimit": ["@koshnic/ratelimit@1.0.3", "", { "dependencies": { "@types/chai": "^4.3.9", "@types/mocha": "^10.0.3", "chai": "^4.3.10", "ioredis": "^5.3.2", "mocha": "^10.2.0" } }, "sha512-cfDcSc+I+M4hNM+/4M+lfn8UuTq4OEFKl78ThOcGNaO7g8tWb1vm2qVpV1p1loYao1mqk00NBNwHQu2E/qFq2g=="], + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], @@ -371,6 +373,8 @@ "@types/bun": ["@types/bun@1.2.11", "", { "dependencies": { "bun-types": "1.2.11" } }, "sha512-ZLbbI91EmmGwlWTRWuV6J19IUiUC5YQ3TCEuSHI3usIP75kuoA8/0PVF+LTrbEnVc8JIhpElWOxv1ocI1fJBbw=="], + "@types/chai": ["@types/chai@4.3.20", "", {}, "sha512-/pC9HAB5I/xMlc5FP77qjCnI16ChlJfW0tGa0IUcFn38VJrTV6DeZ60NU5KZBtaOZqjdpwTWohz5HU1RrhiYxQ=="], + "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], "@types/estree": ["@types/estree@1.0.7", "", {}, "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ=="], @@ -383,6 +387,8 @@ "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], + "@types/mocha": ["@types/mocha@10.0.10", "", {}, "sha512-xPyYSz1cMPnJQhl0CLMH68j3gprKZaTjG3s5Vi+fDgx+uhG9NOXwbVt52eFS8ECyXhyKcjDLCBEqBExKuiZb7Q=="], + "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], "@types/nlcst": ["@types/nlcst@2.0.3", "", { "dependencies": { "@types/unist": "*" } }, "sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA=="], @@ -419,6 +425,8 @@ "ansi-align": ["ansi-align@3.0.1", "", { "dependencies": { "string-width": "^4.1.0" } }, "sha512-IOfwwBF5iczOjp/WeY4YxyjqAFMQoZufdQWDd19SEExbVLNXqvpzSJ/M7Za4/sCPmQ0+GRquoA7bGcINcxew6w=="], + "ansi-colors": ["ansi-colors@4.1.3", "", {}, "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw=="], + "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], @@ -457,6 +465,8 @@ "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], + "binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="], + "blob-to-buffer": ["blob-to-buffer@1.2.9", "", {}, "sha512-BF033y5fN6OCofD3vgHmNtwZWRcq9NLyyxyILx9hfMy1sXYy4ojFl765hJ2lP0YaN2fuxPaLO2Vzzoxy0FLFFA=="], "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], @@ -465,10 +475,14 @@ "boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="], - "brace-expansion": ["brace-expansion@1.1.11", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA=="], + "brace-expansion": ["brace-expansion@2.0.1", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="], + + "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], "brotli": ["brotli@1.3.3", "", { "dependencies": { "base64-js": "^1.1.2" } }, "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg=="], + "browser-stdout": ["browser-stdout@1.3.1", "", {}, "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw=="], + "browserslist": ["browserslist@4.24.4", "", { "dependencies": { "caniuse-lite": "^1.0.30001688", "electron-to-chromium": "^1.5.73", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.1" }, "bin": { "browserslist": "cli.js" } }, "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A=="], "bullmq": ["bullmq@5.52.1", "", { "dependencies": { "cron-parser": "^4.9.0", "ioredis": "^5.4.1", "msgpackr": "^1.11.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-u7CSV9wID3MBEX2DNubEErbAlrADgm8abUBAi6h8rQTnuTkhhgMs2iD7uhqplK8lIgUOkBIW3sDJWaMSInH47A=="], @@ -501,7 +515,7 @@ "check-error": ["check-error@2.1.1", "", {}, "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw=="], - "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], + "chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="], "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], @@ -569,6 +583,8 @@ "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], + "decamelize": ["decamelize@4.0.0", "", {}, "sha512-9iE1PgSik9HeIIw2JO94IidnE3eBoQrFJ3w7sFuzSX4DpmZ3v5sZpUiV5Swcf6mQEF+Y0ru8Neo+p+nyh2J+hQ=="], + "decode-named-character-reference": ["decode-named-character-reference@1.1.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w=="], "dedent-js": ["dedent-js@1.0.1", "", {}, "sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ=="], @@ -679,8 +695,14 @@ "filelist": ["filelist@1.0.4", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q=="], + "fill-range": ["fill-range@7.1.1", "", { "dependencies": { "to-regex-range": "^5.0.1" } }, "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg=="], + "finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="], + "find-up": ["find-up@5.0.0", "", { "dependencies": { "locate-path": "^6.0.0", "path-exists": "^4.0.0" } }, "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng=="], + + "flat": ["flat@5.0.2", "", { "bin": { "flat": "cli.js" } }, "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ=="], + "flatbuffers": ["flatbuffers@25.2.10", "", {}, "sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw=="], "flattie": ["flattie@1.1.1", "", {}, "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ=="], @@ -701,6 +723,8 @@ "frontend": ["frontend@workspace:packages/frontend"], + "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], @@ -709,13 +733,17 @@ "get-east-asian-width": ["get-east-asian-width@1.3.0", "", {}, "sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ=="], + "get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="], + "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], "github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="], - "glob": ["glob@10.4.5", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg=="], + "glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], + + "glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "glob-regex": ["glob-regex@0.3.2", "", {}, "sha512-m5blUd3/OqDTWwzBBtWBPrGlAzatRywHameHeekAZyZrskYouOGdNB8T/q6JucucvJXtOuyHIn0/Yia7iDasDw=="], @@ -763,6 +791,8 @@ "hastscript": ["hastscript@9.0.1", "", { "dependencies": { "@types/hast": "^3.0.0", "comma-separated-tokens": "^2.0.0", "hast-util-parse-selector": "^4.0.0", "property-information": "^7.0.0", "space-separated-tokens": "^2.0.0" } }, "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w=="], + "he": ["he@1.2.0", "", { "bin": { "he": "bin/he" } }, "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="], + "hono": ["hono@4.7.8", "", {}, "sha512-PCibtFdxa7/Ldud9yddl1G81GjYaeMYYTq4ywSaNsYbB1Lug4mwtOMJf2WXykL0pntYwmpRJeOI3NmoDgD+Jxw=="], "hono-rate-limiter": ["hono-rate-limiter@0.4.2", "", { "peerDependencies": { "hono": "^4.1.1" } }, "sha512-AAtFqgADyrmbDijcRTT/HJfwqfvhalya2Zo+MgfdrMPas3zSMD8SU03cv+ZsYwRU1swv7zgVt0shwN059yzhjw=="], @@ -779,6 +809,8 @@ "import-meta-resolve": ["import-meta-resolve@4.1.0", "", {}, "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw=="], + "inflight": ["inflight@1.0.6", "", { "dependencies": { "once": "^1.3.0", "wrappy": "1" } }, "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA=="], + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], "ioredis": ["ioredis@5.6.1", "", { "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA=="], @@ -789,13 +821,21 @@ "is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="], + "is-binary-path": ["is-binary-path@2.1.0", "", { "dependencies": { "binary-extensions": "^2.0.0" } }, "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw=="], + "is-docker": ["is-docker@3.0.0", "", { "bin": { "is-docker": "cli.js" } }, "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ=="], + "is-extglob": ["is-extglob@2.1.1", "", {}, "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ=="], + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + "is-glob": ["is-glob@4.0.3", "", { "dependencies": { "is-extglob": "^2.1.1" } }, "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg=="], + "is-inside-container": ["is-inside-container@1.0.0", "", { "dependencies": { "is-docker": "^3.0.0" }, "bin": { "is-inside-container": "cli.js" } }, "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA=="], - "is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + "is-number": ["is-number@7.0.0", "", {}, "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng=="], + + "is-plain-obj": ["is-plain-obj@2.1.0", "", {}, "sha512-YWnfyRwxL/+SsrWYfOpUtz5b3YD+nyfkHvjbcanzk8zgyO4ASD67uVMRt8k5bM4lLMDnXfriRhOpemw+NfT1eA=="], "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], @@ -805,6 +845,8 @@ "is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="], + "is-unicode-supported": ["is-unicode-supported@0.1.0", "", {}, "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw=="], + "is-wsl": ["is-wsl@3.1.0", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw=="], "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], @@ -815,8 +857,6 @@ "jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="], - "jose": ["jose@6.0.11", "", {}, "sha512-QxG7EaliDARm1O1S8BGakqncGT9s25bKL1WSf6/oa17Tkqwi8D2ZNglqCF+DsYF88/rV66Q/Q2mFAy697E1DUg=="], - "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], @@ -855,12 +895,16 @@ "locate-character": ["locate-character@3.0.0", "", {}, "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="], + "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], + "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], + "log-symbols": ["log-symbols@4.1.0", "", { "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" } }, "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg=="], + "logform": ["logform@2.7.0", "", { "dependencies": { "@colors/colors": "1.6.0", "@types/triple-beam": "^1.3.2", "fecha": "^4.2.0", "ms": "^2.1.1", "safe-stable-stringify": "^2.3.1", "triple-beam": "^1.3.0" } }, "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ=="], "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], @@ -977,7 +1021,7 @@ "mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="], - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + "minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], @@ -987,6 +1031,8 @@ "mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="], + "mocha": ["mocha@10.8.2", "", { "dependencies": { "ansi-colors": "^4.1.3", "browser-stdout": "^1.3.1", "chokidar": "^3.5.3", "debug": "^4.3.5", "diff": "^5.2.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", "glob": "^8.1.0", "he": "^1.2.0", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", "minimatch": "^5.1.6", "ms": "^2.1.3", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", "workerpool": "^6.5.1", "yargs": "^16.2.0", "yargs-parser": "^20.2.9", "yargs-unparser": "^2.0.0" }, "bin": { "mocha": "bin/mocha.js", "_mocha": "bin/_mocha" } }, "sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg=="], + "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -1053,6 +1099,8 @@ "p-limit": ["p-limit@6.2.0", "", { "dependencies": { "yocto-queue": "^1.1.1" } }, "sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA=="], + "p-locate": ["p-locate@5.0.0", "", { "dependencies": { "p-limit": "^3.0.2" } }, "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw=="], + "p-queue": ["p-queue@8.1.0", "", { "dependencies": { "eventemitter3": "^5.0.1", "p-timeout": "^6.1.2" } }, "sha512-mxLDbbGIBEXTJL0zEx8JIylaj3xQ7Z/7eEVjcF9fJX4DBiH9oqe+oahYnlKKxm0Ci9TlWTyhSHgygxMxjIB2jw=="], "p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="], @@ -1071,6 +1119,8 @@ "pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="], + "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], "path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], @@ -1141,6 +1191,8 @@ "radix3": ["radix3@1.1.2", "", {}, "sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA=="], + "randombytes": ["randombytes@2.1.0", "", { "dependencies": { "safe-buffer": "^5.1.0" } }, "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ=="], + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], "rate-limit-redis": ["rate-limit-redis@4.2.0", "", { "peerDependencies": { "express-rate-limit": ">= 6" } }, "sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA=="], @@ -1149,7 +1201,7 @@ "readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="], - "readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], + "readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="], "recrawl-sync": ["recrawl-sync@2.2.3", "", { "dependencies": { "@cush/relative": "^1.0.0", "glob-regex": "^0.3.0", "slash": "^3.0.0", "sucrase": "^3.20.3", "tslib": "^1.9.3" } }, "sha512-vSaTR9t+cpxlskkdUFrsEpnf67kSmPk66yAGT1fZPrDudxQjoMzPgQhSMImQ0pAw5k0NPirefQfhopSjhdUtpQ=="], @@ -1217,6 +1269,8 @@ "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + "serialize-javascript": ["serialize-javascript@6.0.2", "", { "dependencies": { "randombytes": "^2.1.0" } }, "sha512-Saa1xPByTTq2gdeFZYLLo+RFE35NHZkAbqZeWNd3BpzppeVisAqpDjcp8dyf6uIvEqJRd46jemmyA4iFIeVk8g=="], + "serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="], "server-destroy": ["server-destroy@1.0.1", "", {}, "sha512-rb+9B5YBIEzYcD6x2VKidaa+cqYBJQKnU4oe4E3ANwRRN56yk/ua1YCJT1n21NTS8w6CcOclAKNP3PhdCXKYtQ=="], @@ -1285,6 +1339,8 @@ "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], + "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], + "sucrase": ["sucrase@3.35.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", "commander": "^4.0.0", "glob": "^10.3.10", "lines-and-columns": "^1.1.6", "mz": "^2.7.0", "pirates": "^4.0.1", "ts-interface-checker": "^0.1.9" }, "bin": { "sucrase": "bin/sucrase", "sucrase-node": "bin/sucrase-node" } }, "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA=="], "supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], @@ -1321,6 +1377,8 @@ "tinyspy": ["tinyspy@3.0.2", "", {}, "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q=="], + "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], "toposort": ["toposort@2.0.2", "", {}, "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg=="], @@ -1343,6 +1401,8 @@ "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="], + "type-fest": ["type-fest@2.19.0", "", {}, "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="], "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], @@ -1435,6 +1495,8 @@ "winston-transport": ["winston-transport@4.9.0", "", { "dependencies": { "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" } }, "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A=="], + "workerpool": ["workerpool@6.5.1", "", {}, "sha512-Fs4dNYcsdpYSAfVxhnl1L5zTksjvOJxtC5hzMNl+1t9B8hTJTdKDyZ5ju7ztgPy+ft9tBFXoOlDNiOT9WUXZlA=="], + "wrap-ansi": ["wrap-ansi@9.0.0", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q=="], "wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], @@ -1453,6 +1515,8 @@ "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], + "yargs-unparser": ["yargs-unparser@2.0.0", "", { "dependencies": { "camelcase": "^6.0.0", "decamelize": "^4.0.0", "flat": "^5.0.2", "is-plain-obj": "^2.1.0" } }, "sha512-7pRTIA9Qc1caZ0bZ6RYRGbHJthJWuakf+WmHK0rVeLkNrrGhfoabBNdue6kdINI6r4if7ocq9aD/n7xwKOdzOA=="], + "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="], "yocto-spinner": ["yocto-spinner@0.2.2", "", { "dependencies": { "yoctocolors": "^2.1.1" } }, "sha512-21rPcM3e4vCpOXThiFRByX8amU5By1R0wNS8Oex+DP3YgC8xdU0vEJ/K8cbPLiIJVosSSysgcFof6s6MSD5/Vw=="], @@ -1479,6 +1543,8 @@ "@isaacs/cliui/wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], + "@koshnic/ratelimit/chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="], + "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.4.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.0.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g=="], @@ -1509,28 +1575,42 @@ "concurrently/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], - "filelist/minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="], - "form-data/mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], - "hast-util-to-parse5/property-information": ["property-information@6.5.0", "", {}, "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig=="], "jake/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "jake/minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + + "log-symbols/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], + "mocha/yargs": ["yargs@16.2.0", "", { "dependencies": { "cliui": "^7.0.2", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.0", "y18n": "^5.0.5", "yargs-parser": "^20.2.2" } }, "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw=="], + + "mocha/yargs-parser": ["yargs-parser@20.2.9", "", {}, "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w=="], + "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.22.0-dev.20250409-89f8206ba4", "", {}, "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ=="], + "p-locate/p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], + "pg/pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], "prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], + "readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "recrawl-sync/tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], "serialize-error/type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], + "sucrase/glob": ["glob@10.4.5", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg=="], + + "unified/is-plain-obj": ["is-plain-obj@4.1.0", "", {}, "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg=="], + + "unstorage/chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], + "widest-line/string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], "wrap-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], @@ -1539,6 +1619,8 @@ "wrap-ansi/strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="], + "yargs-unparser/camelcase": ["camelcase@6.3.0", "", {}, "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA=="], + "@huggingface/transformers/onnxruntime-node/onnxruntime-common": ["onnxruntime-common@1.21.0", "", {}, "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ=="], "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], @@ -1547,6 +1629,16 @@ "@isaacs/cliui/wrap-ansi/ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], + "@koshnic/ratelimit/chai/assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="], + + "@koshnic/ratelimit/chai/check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="], + + "@koshnic/ratelimit/chai/deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="], + + "@koshnic/ratelimit/chai/loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], + + "@koshnic/ratelimit/chai/pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="], + "astro/sharp/@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.0.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ=="], "astro/sharp/@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.33.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.0.4" }, "os": "darwin", "cpu": "x64" }, "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q=="], @@ -1593,14 +1685,18 @@ "concurrently/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], - "filelist/minimatch/brace-expansion": ["brace-expansion@2.0.1", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="], - "form-data/mime-types/mime-db": ["mime-db@1.52.0", "", {}, "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="], - "glob/minimatch/brace-expansion": ["brace-expansion@2.0.1", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="], - "jake/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + "jake/minimatch/brace-expansion": ["brace-expansion@1.1.11", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA=="], + + "log-symbols/chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "mocha/yargs/cliui": ["cliui@7.0.4", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", "wrap-ansi": "^7.0.0" } }, "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ=="], + + "p-locate/p-limit/yocto-queue": ["yocto-queue@0.1.0", "", {}, "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q=="], + "pg/pg-types/postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "pg/pg-types/postgres-bytea": ["postgres-bytea@1.0.0", "", {}, "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="], @@ -1609,6 +1705,10 @@ "pg/pg-types/postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + "sucrase/glob/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + + "unstorage/chokidar/readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], + "widest-line/string-width/emoji-regex": ["emoji-regex@10.4.0", "", {}, "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw=="], "widest-line/string-width/strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="], @@ -1621,6 +1721,8 @@ "colorspace/color/color-convert/color-name": ["color-name@1.1.3", "", {}, "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="], + "mocha/yargs/cliui/wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + "widest-line/string-width/strip-ansi/ansi-regex": ["ansi-regex@6.1.0", "", {}, "sha512-7HSX4QQb4CspciLpVFwyRe79O3xsIZDDLER21kERQ71oaPodF8jL725AgJMFAYbooIqolJoRLuM81SpeUkpkvA=="], } } diff --git a/packages/backend/lib/auth/captchaDifficulty.ts b/packages/backend/lib/auth/captchaDifficulty.ts index 582fae7..4dd270e 100644 --- a/packages/backend/lib/auth/captchaDifficulty.ts +++ b/packages/backend/lib/auth/captchaDifficulty.ts @@ -1,6 +1,8 @@ import { Psql } from "@core/db/psql"; import { SlidingWindow } from "@core/mq/slidingWindow.ts"; import { redis } from "@core/db/redis.ts"; +import { getIdentifier } from "@/middleware/rateLimiters.ts"; +import { Context } from "hono"; type seconds = number; @@ -33,7 +35,9 @@ export const getCaptchaConfigMaxDuration = async (sql: Psql, route: string): Pro } -export const getCurrentCaptchaDifficulty = async (sql: Psql, route: string): Promise => { +export const getCurrentCaptchaDifficulty = async (sql: Psql, c: Context | string): Promise => { + const isRoute = typeof c === "string"; + const route = isRoute ? c : `${c.req.method}-${c.req.path}` const configs = await getCaptchaDifficultyConfigByRoute(sql, route); if (configs.length < 1) { return null @@ -44,14 +48,15 @@ export const getCurrentCaptchaDifficulty = async (sql: Psql, route: string): Pro const maxDuration = configs.reduce((max, config) => Math.max(max, config.duration), 0); const slidingWindow = new SlidingWindow(redis, maxDuration); - for (let i = 0; i < configs.length; i++) { + for (let i = 1; i < configs.length; i++) { const config = configs[i]; const lastConfig = configs[i - 1]; - const count = await slidingWindow.count(`captcha-${route}`, config.duration); + const identifier = isRoute ? c : getIdentifier(c, config.global); + const count = await slidingWindow.count(`captcha-${identifier}`, config.duration); if (count >= config.threshold) { continue; } return lastConfig.difficulty } - return configs[0].difficulty; + return configs[configs.length-1].difficulty; } diff --git a/packages/backend/middleware/captcha.ts b/packages/backend/middleware/captcha.ts index ceef077..7f9e55e 100644 --- a/packages/backend/middleware/captcha.ts +++ b/packages/backend/middleware/captcha.ts @@ -9,6 +9,7 @@ import { JwtTokenInvalid, JwtTokenExpired } from "hono/utils/jwt/types"; import { getJWTsecret } from "@/lib/auth/getJWTsecret.ts"; import { lockManager } from "@core/mq/lockManager.ts"; import { object, string, number, ValidationError } from "yup"; +import { getIdentifier } from "@/middleware/rateLimiters.ts"; const tokenSchema = object({ exp: number().integer(), @@ -48,7 +49,7 @@ export const captchaMiddleware = async (c: Context, next: Next) => { const method = c.req.method; const route = `${method}-${path}`; - const requiredDifficulty = await getCurrentCaptchaDifficulty(sqlCred, route); + const requiredDifficulty = await getCurrentCaptchaDifficulty(sqlCred, c); try { const decodedPayload = await verify(token, jwtSecret); @@ -65,7 +66,7 @@ export const captchaMiddleware = async (c: Context, next: Next) => { } if (difficulty < requiredDifficulty) { const response: ErrorResponse = { - message: "Token to weak.", + message: "Token too weak.", code: "UNAUTHORIZED" }; return c.json(response, 401); @@ -106,7 +107,11 @@ export const captchaMiddleware = async (c: Context, next: Next) => { } const duration = await getCaptchaConfigMaxDuration(sqlCred, route); const window = new SlidingWindow(redis, duration); - await window.event(`captcha-${route}`); + + const identifierWithIP = getIdentifier(c, true); + const identifier = getIdentifier(c, false); + await window.event(`captcha-${identifier}`); + await window.event(`captcha-${identifierWithIP}`); await next(); }; \ No newline at end of file diff --git a/packages/backend/middleware/rateLimiters.ts b/packages/backend/middleware/rateLimiters.ts index e937406..53c4115 100644 --- a/packages/backend/middleware/rateLimiters.ts +++ b/packages/backend/middleware/rateLimiters.ts @@ -1,11 +1,10 @@ -import { rateLimiter, Store } from "hono-rate-limiter"; import type { BlankEnv } from "hono/types"; -import { MINUTE } from "@core/const/time.ts"; import { getConnInfo } from "hono/bun"; -import type { Context } from "hono"; -import { redis } from "@core/db/redis.ts"; -import { RedisStore } from "rate-limit-redis"; +import { Context, Next } from "hono"; import { generateRandomId } from "@core/lib/randomID.ts"; +import { RateLimiter } from "@koshnic/ratelimit"; +import { ErrorResponse } from "@/src/schema"; +import { redis } from "@core/db/redis.ts"; export const getIdentifier = (c: Context, includeIP: boolean = true) => { let ipAddr = generateRandomId(6); @@ -20,16 +19,26 @@ export const getIdentifier = (c: Context, includeIP: boolean = true) => { const path = c.req.path; const method = c.req.method; const ipIdentifier = includeIP ? `@${ipAddr}` : ""; - return `${method}-${path}${ipIdentifier}` -} + return `${method}-${path}${ipIdentifier}`; +}; -export const registerRateLimiter = rateLimiter({ - windowMs: 60 * MINUTE, - limit: 10, - standardHeaders: "draft-6", - keyGenerator: getIdentifier, - store: new RedisStore({ - // @ts-expect-error - Known issue: the `c`all` function is not present in @types/ioredis - sendCommand: (...args: string[]) => redis.call(...args) - }) as unknown as Store -}); \ No newline at end of file +export const registerRateLimiter = async (c: Context, next: Next) => { + const limiter = new RateLimiter(redis); + const identifier = getIdentifier(c, true); + const { allowed, retryAfter } = await limiter.allow(identifier, { + burst: 5, + ratePerPeriod: 5, + period: 120, + cost: 1 + }); + + if (!allowed) { + const response: ErrorResponse = { + message: `Too many requests, please retry after ${Math.round(retryAfter)} seconds.`, + code: "RATE_LIMIT_EXCEEDED" + }; + return c.json(response, 429); + } + + await next(); +}; \ No newline at end of file diff --git a/packages/backend/routes/captcha/session/POST.ts b/packages/backend/routes/captcha/session/POST.ts index 7d2c3d1..2686a77 100644 --- a/packages/backend/routes/captcha/session/POST.ts +++ b/packages/backend/routes/captcha/session/POST.ts @@ -1,6 +1,21 @@ import { createHandlers } from "src/utils.ts"; +import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; +import { sqlCred } from "@core/db/dbNew.ts"; +import { object, string, ValidationError } from "yup"; +import { ErrorResponse } from "@/src/schema"; +import type { ContentfulStatusCode } from "hono/utils/http-status"; -const DIFFICULTY = 200000; +const bodySchema = object({ + route: string().matches(/(?:GET|POST|PUT|PATCH|DELETE)-\/.*/g) +}); + +interface CaptchaSessionResponse { + success: boolean; + id: string; + g: string; + n: string; + t: number; +} const createNewChallenge = async (difficulty: number) => { const baseURL = process.env["UCAPTCHA_URL"]; @@ -17,7 +32,28 @@ const createNewChallenge = async (difficulty: number) => { }); } -export const createCaptchaSessionHandler = createHandlers(async (_c) => { - const res = await createNewChallenge(DIFFICULTY); - return res; +export const createCaptchaSessionHandler = createHandlers(async (c) => { + try { + const requestBody = await bodySchema.validate(await c.req.json()); + const { route } = requestBody; + const difficuly = await getCurrentCaptchaDifficulty(sqlCred, route) + const res = await createNewChallenge(difficuly); + return c.json(await res.json(), res.status as ContentfulStatusCode); + } catch (e: unknown) { + if (e instanceof ValidationError) { + const response: ErrorResponse = { + code: "INVALID_QUERY_PARAMS", + message: "Invalid query parameters", + errors: e.errors + }; + return c.json(response, 400); + } else { + const response: ErrorResponse = { + code: "UNKNOWN_ERROR", + message: "Unknown error", + errors: [e] + }; + return c.json>(response, 500); + } + } }); diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index 4a50d7b..5b16743 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -8,7 +8,8 @@ type ErrorCode = | "UNAUTHORIZED" | "INVALID_CREDENTIALS" | "ENTITY_NOT_FOUND" - | "SERVER_ERROR"; + | "SERVER_ERROR" + | "RATE_LIMIT_EXCEEDED"; export interface ErrorResponse { code: ErrorCode; diff --git a/packages/core/mq/multipleRateLimiter.ts b/packages/core/mq/multipleRateLimiter.ts new file mode 100644 index 0000000..c5f7bd1 --- /dev/null +++ b/packages/core/mq/multipleRateLimiter.ts @@ -0,0 +1,63 @@ +import { RateLimiter as Limiter } from "@koshnic/ratelimit"; +import { redis } from "@core/db/redis.ts"; + +export interface RateLimiterConfig { + duration: number; + max: number; +} + +export class MultipleRateLimiter { + private readonly name: string; + private readonly configs: RateLimiterConfig[] = []; + private readonly limiter: Limiter; + + /* + * @param name The name of the rate limiter + * @param configs The configuration of the rate limiter, containing: + * - duration: The duration of window in seconds + * - max: The maximum number of tokens allowed in the window + */ + constructor( + name: string, + configs: RateLimiterConfig[] + ) { + this.configs = configs; + this.limiter = new Limiter(redis); + this.name = name; + } + + /* + * Check if the event has reached the rate limit + */ + async getAvailability(): Promise { + for (let i = 0; i < this.configs.length; i++) { + const { duration, max } = this.configs[i]; + const { remaining } = await this.limiter.allow(`cvsa:${this.name}_${i}`, { + burst: max, + ratePerPeriod: max, + period: duration, + cost: 0 + }); + + if (remaining < 1) { + return false; + } + } + return true; + } + + /* + * Trigger an event in the rate limiter + */ + async trigger(): Promise { + for (let i = 0; i < this.configs.length; i++) { + const { duration, max } = this.configs[i]; + await this.limiter.allow(`cvsa:${this.name}_${i}`, { + burst: max, + ratePerPeriod: max, + period: duration, + cost: 1 + }); + } + } +} \ No newline at end of file diff --git a/packages/core/mq/rateLimiter.ts b/packages/core/mq/rateLimiter.ts deleted file mode 100644 index d15ddd5..0000000 --- a/packages/core/mq/rateLimiter.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { TokenBucket } from "./tokenBucket.ts"; - -export interface RateLimiterConfig { - duration: number; - max: number; -} - -export class RateLimiter { - private configs: RateLimiterConfig[] = []; - private buckets: TokenBucket[] = []; - private readonly identifierFn: (configIndex: number) => string; - - /* - * @param name The name of the rate limiter - * @param configs The configuration of the rate limiter, containing: - * - tokenBucket: The token bucket instance - * - max: The maximum number of tokens allowed per operation - */ - constructor( - name: string, - configs: RateLimiterConfig[], - identifierFn?: (configIndex: number) => string - ) { - this.configs = configs; - this.identifierFn = identifierFn || ((index) => `${name}_config_${index}`); - for (let i = 0; i < configs.length; i++) { - const config = configs[i]; - const bucket = new TokenBucket({ - max: config.max, - duration: config.duration, - identifier: this.identifierFn(i), - }) - this.buckets.push(bucket); - } - } - - /* - * Check if the event has reached the rate limit - */ - async getAvailability(): Promise { - for (let i = 0; i < this.configs.length; i++) { - const remaining = await this.buckets[i].getRemainingTokens(); - - if (remaining === null) { - return false; // Rate limit exceeded - } - } - return true; - } - - /* - * Trigger an event in the rate limiter - */ - async trigger(): Promise { - for (let i = 0; i < this.configs.length; i++) { - await this.buckets[i].consume(1); - } - } - - /* - * Clear all buckets for all configurations - */ - async clear(): Promise { - for (let i = 0; i < this.configs.length; i++) { - await this.buckets[i].reset(); - } - } -} \ No newline at end of file diff --git a/packages/core/mq/tokenBucket.ts b/packages/core/mq/tokenBucket.ts deleted file mode 100644 index e56d368..0000000 --- a/packages/core/mq/tokenBucket.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { redis } from "@core/db/redis"; -import { SECOND } from "@core/const/time"; - -export interface TokenBucketRateOptions { - capacity: number; - rate: number; - identifier: string; - keyPrefix?: string; -} - -export interface TokenBucketDurationOptions { - duration: number; - max: number; - identifier: string; - keyPrefix?: string; -} - -export type TokenBucketConstructorOptions = TokenBucketRateOptions | TokenBucketDurationOptions; - -export class TokenBucket { - private readonly capacity: number; - private readonly rate: number; - private readonly keyPrefix: string; - private readonly identifier: string; - - constructor(options: TokenBucketConstructorOptions) { - if (!options.identifier) { - throw new Error("Identifier is required."); - } - this.identifier = options.identifier; - this.keyPrefix = options.keyPrefix || "cvsa:token_bucket:"; - - const isRateOptions = 'capacity' in options && 'rate' in options; - const isDurationOptions = 'duration' in options && 'max' in options; - - if (isRateOptions && isDurationOptions) { - throw new Error("Provide either 'capacity'/'rate' or 'duration'/'max', not both."); - } else if (isRateOptions) { - if (options.capacity <= 0 || options.rate <= 0) { - throw new Error("'capacity' and 'rate' must be greater than zero."); - } - this.capacity = options.capacity; - this.rate = options.rate; - } else if (isDurationOptions) { - if (options.duration <= 0 || options.max <= 0) { - throw new Error("'duration' and 'max' must be greater than zero."); - } - this.capacity = options.max; - this.rate = options.max / options.duration; - - } else { - throw new Error("Provide either 'capacity'/'rate' or 'duration'/'max'."); - } - } - - getKey(): string { - return `${this.keyPrefix}${this.identifier}`; - } - - /** - * Try to consume a specified number of tokens - * @param count The number of tokens to be consumed - * @returns If consumption is successful, returns the number of remaining tokens; otherwise returns null - */ - public async consume(count: number): Promise { - const key = this.getKey(); - const now = Math.floor(Date.now() / SECOND); - - const script = ` - local tokens_key = KEYS[1] - local last_refilled_key = KEYS[2] - local now = tonumber(ARGV[1]) - local count = tonumber(ARGV[2]) - local capacity = tonumber(ARGV[3]) - local rate = tonumber(ARGV[4]) - - local last_refilled = tonumber(redis.call('GET', last_refilled_key)) or now - local current_tokens = tonumber(redis.call('GET', tokens_key)) or capacity - - local elapsed = now - last_refilled - local new_tokens = elapsed * rate - current_tokens = math.min(capacity, current_tokens + new_tokens) - - if current_tokens >= count then - current_tokens = current_tokens - count - redis.call('SET', tokens_key, current_tokens) - redis.call('SET', last_refilled_key, now) - return current_tokens - else - return nil - end - `; - - const keys = [`${key}:tokens`, `${key}:last_refilled`]; - const args = [now, count, this.capacity, this.rate]; - - const result = await redis.eval(script, keys.length, ...keys, ...args); - - return result as number | null; - } - - public async getRemainingTokens(): Promise { - const key = this.getKey(); - const tokens = await redis.get(`${key}:tokens`); - return Number(tokens) || this.capacity; - } - - public async reset(): Promise { - const key = this.getKey(); - await redis.del(`${key}:tokens`, `${key}:last_refilled`); - } -} diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index 6d7d6d7..a89934d 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,5 +1,5 @@ import logger from "@core/log/logger.ts"; -import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts"; +import { MultipleRateLimiter, type RateLimiterConfig } from "@core/mq/multipleRateLimiter.ts"; import { ReplyError } from "ioredis"; import { SECOND } from "@core/const/time.ts"; import { spawn, SpawnOptions } from "child_process"; @@ -71,11 +71,11 @@ export class NetSchedulerError extends Error { } type LimiterMap = { - [name: string]: RateLimiter; + [name: string]: MultipleRateLimiter; }; type OptionalLimiterMap = { - [name: string]: RateLimiter | null; + [name: string]: MultipleRateLimiter | null; }; type TaskMap = { @@ -119,7 +119,7 @@ class NetworkDelegate { const proxies = this.getTaskProxies(taskName); for (const proxyName of proxies) { const limiterId = "proxy-" + proxyName + "-" + taskName; - this.proxyLimiters[limiterId] = config ? new RateLimiter(limiterId, config) : null; + this.proxyLimiters[limiterId] = config ? new MultipleRateLimiter(limiterId, config) : null; } } @@ -147,7 +147,7 @@ class NetworkDelegate { } for (const proxyName of bindProxies) { const limiterId = "provider-" + proxyName + "-" + providerName; - this.providerLimiters[limiterId] = new RateLimiter(limiterId, config); + this.providerLimiters[limiterId] = new MultipleRateLimiter(limiterId, config); } } diff --git a/packages/core/package.json b/packages/core/package.json index 9c291cb..f8ada58 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -4,6 +4,7 @@ "test": "bun --env-file=.env.test run vitest" }, "dependencies": { + "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", From 6d946f74dfa6342d525857bdd5e52b0a2a0f2fb0 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 18 May 2025 03:52:42 +0800 Subject: [PATCH 12/12] update: rate limiter --- packages/core/mq/multipleRateLimiter.ts | 36 +++++++--------- packages/core/net/delegate.ts | 55 +++++++------------------ 2 files changed, 29 insertions(+), 62 deletions(-) diff --git a/packages/core/mq/multipleRateLimiter.ts b/packages/core/mq/multipleRateLimiter.ts index c5f7bd1..5c9333e 100644 --- a/packages/core/mq/multipleRateLimiter.ts +++ b/packages/core/mq/multipleRateLimiter.ts @@ -6,6 +6,15 @@ export interface RateLimiterConfig { max: number; } +export class RateLimiterError extends Error { + public code: string; + constructor(message: string) { + super(message); + this.name = "RateLimiterError"; + this.code = "RATE_LIMIT_EXCEEDED"; + } +} + export class MultipleRateLimiter { private readonly name: string; private readonly configs: RateLimiterConfig[] = []; @@ -26,38 +35,21 @@ export class MultipleRateLimiter { this.name = name; } - /* - * Check if the event has reached the rate limit - */ - async getAvailability(): Promise { - for (let i = 0; i < this.configs.length; i++) { - const { duration, max } = this.configs[i]; - const { remaining } = await this.limiter.allow(`cvsa:${this.name}_${i}`, { - burst: max, - ratePerPeriod: max, - period: duration, - cost: 0 - }); - - if (remaining < 1) { - return false; - } - } - return true; - } - /* * Trigger an event in the rate limiter */ - async trigger(): Promise { + async trigger(shouldThrow = true): Promise { for (let i = 0; i < this.configs.length; i++) { const { duration, max } = this.configs[i]; - await this.limiter.allow(`cvsa:${this.name}_${i}`, { + const { allowed } = await this.limiter.allow(`cvsa:${this.name}_${i}`, { burst: max, ratePerPeriod: max, period: duration, cost: 1 }); + if (!allowed && shouldThrow) { + throw new RateLimiterError("Rate limit exceeded") + } } } } \ No newline at end of file diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index a89934d..0c5d856 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,5 +1,5 @@ import logger from "@core/log/logger.ts"; -import { MultipleRateLimiter, type RateLimiterConfig } from "@core/mq/multipleRateLimiter.ts"; +import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter.ts"; import { ReplyError } from "ioredis"; import { SECOND } from "@core/const/time.ts"; import { spawn, SpawnOptions } from "child_process"; @@ -123,16 +123,19 @@ class NetworkDelegate { } } - async triggerLimiter(task: string, proxy: string): Promise { + async triggerLimiter(task: string, proxy: string, force: boolean = false): Promise { const limiterId = "proxy-" + proxy + "-" + task; const providerLimiterId = "provider-" + proxy + "-" + this.tasks[task].provider; try { - await this.proxyLimiters[limiterId]?.trigger(); - await this.providerLimiters[providerLimiterId]?.trigger(); + await this.proxyLimiters[limiterId]?.trigger(!force); + await this.providerLimiters[providerLimiterId]?.trigger(!force); } catch (e) { const error = e as Error; if (e instanceof ReplyError) { logger.error(error, "redis"); + } else if (e instanceof RateLimiterError) { + // Re-throw it to ensure this.request can catch it + throw e; } logger.warn(`Unhandled error: ${error.message}`, "mq", "proxyRequest"); } @@ -166,9 +169,15 @@ class NetworkDelegate { // find a available proxy const proxiesNames = this.getTaskProxies(task); for (const proxyName of shuffleArray(proxiesNames)) { - if (await this.getProxyAvailability(proxyName, task)) { + try { return await this.proxyRequest(url, proxyName, task, method); } + catch (e) { + if (e instanceof RateLimiterError) { + continue; + } + throw e; + } } throw new NetSchedulerError("No proxy is available currently.", "NO_PROXY_AVAILABLE"); } @@ -200,16 +209,8 @@ class NetworkDelegate { throw new NetSchedulerError(`Proxy "${proxyName}" not found`, "PROXY_NOT_FOUND"); } - if (!force) { - const isAvailable = await this.getProxyAvailability(proxyName, task); - const limiter = "proxy-" + proxyName + "-" + task; - if (!isAvailable) { - throw new NetSchedulerError(`Proxy "${limiter}" is rate limited`, "PROXY_RATE_LIMITED"); - } - } - + await this.triggerLimiter(task, proxyName, force); const result = await this.makeRequest(url, proxy, method); - await this.triggerLimiter(task, proxyName); return result; } @@ -224,32 +225,6 @@ class NetworkDelegate { } } - private async getProxyAvailability(proxyName: string, taskName: string): Promise { - try { - const task = this.tasks[taskName]; - const provider = task.provider; - const proxyLimiterId = "proxy-" + proxyName + "-" + task; - const providerLimiterId = "provider-" + proxyName + "-" + provider; - if (!this.proxyLimiters[proxyLimiterId]) { - const providerLimiter = this.providerLimiters[providerLimiterId]; - return await providerLimiter.getAvailability(); - } - const proxyLimiter = this.proxyLimiters[proxyLimiterId]; - const providerLimiter = this.providerLimiters[providerLimiterId]; - const providerAvailable = await providerLimiter.getAvailability(); - const proxyAvailable = await proxyLimiter.getAvailability(); - return providerAvailable && proxyAvailable; - } catch (e) { - const error = e as Error; - if (e instanceof ReplyError) { - logger.error(error, "redis"); - return false; - } - logger.error(error, "mq", "getProxyAvailability"); - return false; - } - } - private async nativeRequest(url: string, method: string): Promise { try { const controller = new AbortController();