add: login, user profile page & license

package.json

@@ -1,23 +1,12 @@
 {
 	"name": "cvsa",
-	"version": "3.15.34",
+	"version": "2.13.22",
-	"private": false,
+	"private": true,
 	"type": "module",
 	"workspaces": [
 		"packages/frontend",
 		"packages/core",
 		"packages/backend",
 		"packages/crawler"
-	],
+	]
-	"dependencies": {
-		"arg": "^5.0.2",
-		"postgres": "^3.4.5"
-	},
-	"devDependencies": {
-		"@types/bun": "^1.2.15",
-		"prettier": "^3.5.3",
-		"vite-tsconfig-paths": "^5.1.4",
-		"vitest": "^3.1.2",
-		"vitest-tsconfig-paths": "^3.4.1"
-	}
 }

packages/backend/db/latestSnapshots.ts

@@ -1,13 +0,0 @@
-import { sql } from "@core/db/dbNew";
-import type { LatestSnapshotType } from "@core/db/schema.d.ts";
-
-export async function getVideosInViewsRange(minViews: number, maxViews: number) {
-	return sql<LatestSnapshotType[]>`
-        SELECT *
-        FROM latest_video_snapshot
-        WHERE views >= ${minViews}
-        AND views <= ${maxViews}
-        ORDER BY views DESC
-        LIMIT 5000
-	`;
-}

packages/backend/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@cvsa/backend",
 	"private": false,
-	"version": "0.6.0",
+	"version": "0.5.3",
 	"scripts": {
 		"format": "prettier --write .",
 		"dev": "NODE_ENV=development bun run --hot src/main.ts",

packages/backend/routes/session/[id]/DELETE.ts

@@ -1,75 +0,0 @@
-import { Context } from "hono";
-import { Bindings, BlankEnv } from "hono/types";
-import { ErrorResponse } from "src/schema";
-import { createHandlers } from "src/utils.ts";
-import { sqlCred } from "@core/db/dbNew";
-import { object, string, ValidationError } from "yup";
-import { setCookie } from "hono/cookie";
-
-const loginSessionExists = async (sessionID: string) => {
-	const result = await sqlCred`
-        SELECT 1
-        FROM login_sessions
-        WHERE id = ${sessionID}
-	`;
-	return result.length > 0;
-};
-
-export const logoutHandler = createHandlers(async (c: Context<BlankEnv & { Bindings: Bindings }, "/session/:id">) => {
-	try {
-		const session_id = c.req.param("id");
-
-		const exists = loginSessionExists(session_id);
-
-		if (!exists) {
-			const response: ErrorResponse<string> = {
-				message: "Cannot found given session_id.",
-				errors: [`Session ${session_id} not found`],
-				code: "ENTITY_NOT_FOUND"
-			};
-			return c.json<ErrorResponse<string>>(response, 404);
-		}
-
-		await sqlCred`
-		    UPDATE login_sessions
-			SET deactivated_at = CURRENT_TIMESTAMP
-			WHERE id = ${session_id}
-		`;
-
-		const isDev = process.env.NODE_ENV === "development";
-
-		setCookie(c, "session_id", "", {
-			path: "/",
-			maxAge: 0,
-			domain: process.env.DOMAIN,
-			secure: isDev ? true : true,
-			sameSite: isDev ? "None" : "Lax",
-			httpOnly: true
-		});
-
-		return c.body(null, 204);
-	} catch (e) {
-		if (e instanceof ValidationError) {
-			const response: ErrorResponse<string> = {
-				message: "Invalid registration data.",
-				errors: e.errors,
-				code: "INVALID_PAYLOAD"
-			};
-			return c.json<ErrorResponse<string>>(response, 400);
-		} else if (e instanceof SyntaxError) {
-			const response: ErrorResponse<string> = {
-				message: "Invalid JSON payload.",
-				errors: [e.message],
-				code: "INVALID_FORMAT"
-			};
-			return c.json<ErrorResponse<string>>(response, 400);
-		} else {
-			const response: ErrorResponse<string> = {
-				message: "Unknown error.",
-				errors: [(e as Error).message],
-				code: "UNKNOWN_ERROR"
-			};
-			return c.json<ErrorResponse<string>>(response, 500);
-		}
-	}
-});

packages/backend/routes/session/index.ts

@@ -1 +0,0 @@
-export * from "./[id]/DELETE";

packages/backend/routes/videos/GET.ts

@@ -1,65 +0,0 @@
-import type { Context } from "hono";
-import { createHandlers } from "src/utils.ts";
-import type { BlankEnv, BlankInput } from "hono/types";
-import { number, object, ValidationError } from "yup";
-import { ErrorResponse } from "src/schema";
-import { startTime, endTime } from "hono/timing";
-import { getVideosInViewsRange } from "@/db/latestSnapshots";
-
-const SnapshotQueryParamsSchema = object({
-	min_views: number().integer().optional().positive(),
-	max_views: number().integer().optional().positive()
-});
-
-type ContextType = Context<BlankEnv, "/videos", BlankInput>;
-
-export const getVideosHanlder = createHandlers(async (c: ContextType) => {
-	startTime(c, "parse", "Parse the request");
-	try {
-		const queryParams = await SnapshotQueryParamsSchema.validate(c.req.query());
-		const { min_views, max_views } = queryParams;
-
-		if (!min_views && !max_views) {
-			const response: ErrorResponse<string> = {
-				code: "INVALID_QUERY_PARAMS",
-				message: "Invalid query parameters",
-				errors: ["Must provide one of these query parameters: min_views, max_views"]
-			};
-			return c.json<ErrorResponse<string>>(response, 400);
-		}
-
-		endTime(c, "parse");
-
-		startTime(c, "db", "Query the database");
-
-		const minViews = min_views ? min_views : 0;
-		const maxViews = max_views ? max_views : 2147483647;
-
-		const result = await getVideosInViewsRange(minViews, maxViews);
-
-		endTime(c, "db");
-
-		const rows = result.map((row) => ({
-			...row,
-			aid: Number(row.aid)
-		}));
-
-		return c.json(rows);
-	} catch (e: unknown) {
-		if (e instanceof ValidationError) {
-			const response: ErrorResponse<string> = {
-				code: "INVALID_QUERY_PARAMS",
-				message: "Invalid query parameters",
-				errors: e.errors
-			};
-			return c.json<ErrorResponse<string>>(response, 400);
-		} else {
-			const response: ErrorResponse<unknown> = {
-				code: "UNKNOWN_ERROR",
-				message: "Unhandled error",
-				errors: [e]
-			};
-			return c.json<ErrorResponse<unknown>>(response, 500);
-		}
-	}
-});

packages/backend/routes/videos/index.ts

@@ -1 +0,0 @@
-export * from "./GET.ts";

packages/backend/src/main.ts

@@ -15,4 +15,4 @@ configureRoutes(app);
 
 await startServer(app);
 
-export const VERSION = "0.6.0";
+export const VERSION = "0.5.2";

packages/backend/src/routing.ts

@@ -6,23 +6,17 @@ import { Hono } from "hono";
 import { Variables } from "hono/types";
 import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha";
 import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET.ts";
-import { getVideosHanlder } from "@/routes/videos";
 import { loginHandler } from "@/routes/login/session/POST";
-import { logoutHandler } from "@/routes/session";
 
 export function configureRoutes(app: Hono<{ Variables: Variables }>) {
 	app.get("/", ...rootHandler);
 	app.all("/ping", ...pingHandler);
 
-	app.get("/videos", ...getVideosHanlder);
-
 	app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
 	app.get("/video/:id/info", ...videoInfoHandler);
 
 	app.post("/login/session", ...loginHandler);
 
-	app.delete("/session/:id", ...logoutHandler);
-
 	app.post("/user", ...registerHandler);
 	app.get("/user/session/:id", ...getUserByLoginSessionHandler);
 

packages/core/db/schema.d.ts

@@ -8,7 +8,7 @@ export interface BiliUserType {
 
 export interface VideoSnapshotType {
 	id: number;
-	created_at: Date;
+	created_at: string;
 	views: number;
 	coins: number;
 	likes: number;
@@ -35,9 +35,9 @@ export interface SnapshotScheduleType {
 	id: number;
 	aid: number;
 	type?: string;
-	created_at: Date;
+	created_at: string;
-	started_at?: Date;
+	started_at?: string;
-	finished_at?: Date;
+	finished_at?: string;
 	status: string;
 }
 
@@ -48,7 +48,6 @@ export interface UserType {
 	password: string;
 	unq_id: string;
 	role: string;
-	created_at: Date;
 }
 
 export interface BiliVideoMetadataType {

packages/core/package.json

@@ -1,7 +1,7 @@
 {
 	"name": "@cvsa/core",
 	"private": false,
-	"version": "0.0.10",
+	"version": "0.0.7",
 	"scripts": {
 		"test": "bun --env-file=.env.test run vitest",
 		"build": "bun build ./index.ts --target node --outdir ./dist"

packages/crawler/db/bilibili_metadata.ts

@@ -30,7 +30,7 @@ export async function insertVideoLabel(sql: Psql, aid: number, label: number) {
 }
 
 export async function getVideoInfoFromAllData(sql: Psql, aid: number) {
-	const rows = await sql<AllDataType[]>`
+	const rows = await sql<BiliVideoMetadataType[]>`
         SELECT * FROM bilibili_metadata WHERE aid = ${aid}
     `;
 	const row = rows[0];

packages/crawler/db/snapshot.ts

@@ -10,19 +10,19 @@ export async function getVideosNearMilestone(sql: Psql) {
         WHERE
             (views >= 50000 AND views < 100000) OR
             (views >= 900000 AND views < 1000000) OR
-            (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
+            (views >= 9900000 AND views < 10000000)
         UNION
         SELECT ls.*
         FROM latest_video_snapshot ls
         WHERE
             (views >= 90000 AND views < 100000) OR
             (views >= 900000 AND views < 1000000) OR
-            (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
+            (views >= 9900000 AND views < 10000000)
     `;
 	return queryResult.map((row) => {
 		return {
 			...row,
-			aid: Number(row.aid)
+			aid: Number(row.aid),
 		};
 	});
 }
@@ -40,7 +40,7 @@ export async function getLatestVideoSnapshot(sql: Psql, aid: number): Promise<nu
 		return {
 			...row,
 			aid: Number(row.aid),
-			time: new Date(row.time).getTime()
+			time: new Date(row.time).getTime(),
 		};
 	})[0];
 }

packages/crawler/db/snapshotSchedule.ts

@@ -63,6 +63,18 @@ export async function snapshotScheduleExists(sql: Psql, id: number) {
 	return rows.length > 0;
 }
 
+export async function videoHasActiveSchedule(sql: Psql, aid: number) {
+	const rows = await sql<{ status: string }[]>`
+		SELECT status
+		FROM snapshot_schedule 
+		WHERE aid = ${aid}
+			AND (status = 'pending' 
+				OR status = 'processing'
+			)
+	`
+	return rows.length > 0;
+}
+
 export async function videoHasActiveScheduleWithType(sql: Psql, aid: number, type: string) {
 	const rows = await sql<{ status: string }[]>`
 		SELECT status FROM snapshot_schedule 
@@ -79,7 +91,7 @@ export async function videoHasProcessingSchedule(sql: Psql, aid: number) {
 		FROM snapshot_schedule 
 		WHERE aid = ${aid}
 			AND status = 'processing'
-	`;
+	`
 	return rows.length > 0;
 }
 
@@ -90,7 +102,7 @@ export async function bulkGetVideosWithoutProcessingSchedules(sql: Psql, aids: n
 		WHERE aid = ANY(${aids})
 			AND status != 'processing' 
 		GROUP BY aid
-	`;
+	`
 	return rows.map((row) => Number(row.aid));
 }
 
@@ -182,8 +194,7 @@ export async function scheduleSnapshot(
 	aid: number,
 	type: string,
 	targetTime: number,
-	force: boolean = false,
+	force: boolean = false
-	adjustTime: boolean = true
 ) {
 	let adjustedTime = new Date(targetTime);
 	const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type);
@@ -205,7 +216,7 @@ export async function scheduleSnapshot(
 		}
 	}
 	if (hashActiveSchedule && !force) return;
-	if (type !== "milestone" && type !== "new" && adjustTime) {
+	if (type !== "milestone" && type !== "new") {
 		adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis);
 	}
 	logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot");
@@ -225,11 +236,10 @@ export async function bulkScheduleSnapshot(
 	aids: number[],
 	type: string,
 	targetTime: number,
-	force: boolean = false,
+	force: boolean = false
-	adjustTime: boolean = true
 ) {
 	for (const aid of aids) {
-		await scheduleSnapshot(sql, aid, type, targetTime, force, adjustTime);
+		await scheduleSnapshot(sql, aid, type, targetTime, force);
 	}
 }
 
@@ -282,23 +292,23 @@ export async function adjustSnapshotTime(
 }
 
 export async function getSnapshotsInNextSecond(sql: Psql) {
-	return sql<SnapshotScheduleType[]>`
+	const rows = await sql<SnapshotScheduleType[]>`
 		SELECT *
 		FROM snapshot_schedule
-        WHERE started_at <= NOW() + INTERVAL '1 seconds'
+		WHERE started_at <= NOW() + INTERVAL '1 seconds' AND status = 'pending' AND type != 'normal'
-          AND status = 'pending'
+		ORDER BY
-          AND type != 'normal'
+			CASE
-        ORDER BY CASE
 				WHEN type = 'milestone' THEN 0
 				ELSE 1
 			END,
 			started_at
 		LIMIT 10;
-	`;
+	`
+	return rows;
 }
 
 export async function getBulkSnapshotsInNextSecond(sql: Psql) {
-	return sql<SnapshotScheduleType[]>`
+	const rows = await sql<SnapshotScheduleType[]>`
         SELECT *
         FROM snapshot_schedule
         WHERE (started_at <= NOW() + INTERVAL '15 seconds')
@@ -310,34 +320,38 @@ export async function getBulkSnapshotsInNextSecond(sql: Psql) {
                      END,
                  started_at
         LIMIT 1000;
-	`;
+	`
+	return rows;
 }
 
 export async function setSnapshotStatus(sql: Psql, id: number, status: string) {
-	return sql`
+	return await sql`
-        UPDATE snapshot_schedule
+		UPDATE snapshot_schedule SET status = ${status} WHERE id = ${id}
-        SET status = ${status}
-        WHERE id = ${id}
 	`;
 }
 
 export async function bulkSetSnapshotStatus(sql: Psql, ids: number[], status: string) {
-	return sql`
+	return await sql`
-        UPDATE snapshot_schedule
+		UPDATE snapshot_schedule SET status = ${status} WHERE id = ANY(${ids})
-        SET status = ${status}
-        WHERE id = ANY (${ids})
 	`;
 }
 
-export async function getVideosWithoutActiveSnapshotScheduleByType(sql: Psql, type: string) {
+export async function getVideosWithoutActiveSnapshotSchedule(sql: Psql) {
 	const rows = await sql<{ aid: string }[]>`
 		SELECT s.aid
 		FROM songs s
-		LEFT JOIN snapshot_schedule ss ON
+		LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing')
-		    s.aid = ss.aid AND
-		    (ss.status = 'pending' OR ss.status = 'processing') AND
-		    ss.type = ${type}
 		WHERE ss.aid IS NULL
 	`;
 	return rows.map((r) => Number(r.aid));
 }
+
+export async function getAllVideosWithoutActiveSnapshotSchedule(psql: Psql) {
+	const rows = await psql<{ aid: number }[]>`
+		SELECT s.aid
+		FROM bilibili_metadata s
+		LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing')
+		WHERE ss.aid IS NULL
+	`
+	return rows.map((r) => Number(r.aid));
+}

packages/crawler/ml/akari.ts

@@ -16,8 +16,8 @@ class AkariProto extends AIManager {
 	constructor() {
 		super();
 		this.models = {
-			classifier: onnxClassifierPath,
+			"classifier": onnxClassifierPath,
-			embedding: onnxEmbeddingPath
+			"embedding": onnxEmbeddingPath,
 		};
 	}
 
@@ -55,7 +55,7 @@ class AkariProto extends AIManager {
 
 		const { input_ids } = await tokenizer(texts, {
 			add_special_tokens: false,
-			return_tensor: false
+			return_tensor: false,
 		});
 
 		const cumsum = (arr: number[]): number[] =>
@@ -66,9 +66,9 @@ class AkariProto extends AIManager {
 
 		const inputs = {
 			input_ids: new ort.Tensor("int64", new BigInt64Array(flattened_input_ids.map(BigInt)), [
-				flattened_input_ids.length
+				flattened_input_ids.length,
 			]),
-			offsets: new ort.Tensor("int64", new BigInt64Array(offsets.map(BigInt)), [offsets.length])
+			offsets: new ort.Tensor("int64", new BigInt64Array(offsets.map(BigInt)), [offsets.length]),
 		};
 
 		const { embeddings } = await session.run(inputs);
@@ -77,14 +77,21 @@ class AkariProto extends AIManager {
 
 	private async runClassification(embeddings: number[]): Promise<number[]> {
 		const session = this.getModelSession("classifier");
-		const inputTensor = new ort.Tensor(Float32Array.from(embeddings), [1, 3, 1024]);
+		const inputTensor = new ort.Tensor(
+			Float32Array.from(embeddings),
+			[1, 3, 1024],
+		);
 
 		const { logits } = await session.run({ channel_features: inputTensor });
 		return this.softmax(logits.data as Float32Array);
 	}
 
 	public async classifyVideo(title: string, description: string, tags: string, aid?: number): Promise<number> {
-		const embeddings = await this.getJinaEmbeddings1024([title, description, tags]);
+		const embeddings = await this.getJinaEmbeddings1024([
+			title,
+			description,
+			tags,
+		]);
 		const probabilities = await this.runClassification(embeddings);
 		if (aid) {
 			logger.log(`Prediction result for aid: ${aid}: [${probabilities.map((p) => p.toFixed(5))}]`, "ml");

packages/crawler/ml/manager.ts

@@ -6,7 +6,8 @@ export class AIManager {
 	public sessions: { [key: string]: ort.InferenceSession } = {};
 	public models: { [key: string]: string } = {};
 
-	constructor() {}
+	constructor() {
+	}
 
 	public async init() {
 		const modelKeys = Object.keys(this.models);

packages/crawler/mq/exec/archiveSnapshots.ts

@@ -1,28 +1,11 @@
 import { Job } from "bullmq";
-import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
+import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { lockManager } from "@core/mq/lockManager.ts";
 import { getLatestVideoSnapshot } from "db/snapshot.ts";
-import { MINUTE } from "@core/const/time.ts";
+import { HOUR, MINUTE } from "@core/const/time.ts";
 import { sql } from "@core/db/dbNew";
 
-function getNextSaturdayMidnightTimestamp(): number {
-	const now = new Date();
-	const currentDay = now.getDay();
-
-	let daysUntilNextSaturday = (6 - currentDay + 7) % 7;
-
-	if (daysUntilNextSaturday === 0) {
-		daysUntilNextSaturday = 7;
-	}
-
-	const nextSaturday = new Date(now);
-	nextSaturday.setDate(nextSaturday.getDate() + daysUntilNextSaturday);
-	nextSaturday.setHours(0, 0, 0, 0);
-
-	return nextSaturday.getTime();
-}
-
 export const archiveSnapshotsWorker = async (_job: Job) => {
 	try {
 		const startedAt = Date.now();
@@ -31,22 +14,21 @@ export const archiveSnapshotsWorker = async (_job: Job) => {
 			return;
 		}
 		await lockManager.acquireLock("dispatchArchiveSnapshots", 30 * 60);
-		const aids = await getVideosWithoutActiveSnapshotScheduleByType(sql, "archive");
+		const aids = await getAllVideosWithoutActiveSnapshotSchedule(sql);
 		for (const rawAid of aids) {
 			const aid = Number(rawAid);
 			const latestSnapshot = await getLatestVideoSnapshot(sql, aid);
 			const now = Date.now();
 			const lastSnapshotedAt = latestSnapshot?.time ?? now;
-			const nextSatMidnight = getNextSaturdayMidnightTimestamp();
+			const interval = 168;
-			const interval = nextSatMidnight - now;
 			logger.log(
 				`Scheduled archive snapshot for aid ${aid} in ${interval} hours.`,
 				"mq",
 				"fn:archiveSnapshotsWorker"
 			);
-			const targetTime = lastSnapshotedAt + interval;
+			const targetTime = lastSnapshotedAt + interval * HOUR;
 			await scheduleSnapshot(sql, aid, "archive", targetTime);
-			if (now - startedAt > 30 * MINUTE) {
+			if (now - startedAt > 250 * MINUTE) {
 				return;
 			}
 		}

packages/crawler/mq/exec/classifyVideo.ts

@@ -34,7 +34,7 @@ export const classifyVideoWorker = async (job: Job) => {
 
 	await job.updateData({
 		...job.data,
-		label: label
+		label: label,
 	});
 
 	return 0;
@@ -46,19 +46,19 @@ export const classifyVideosWorker = async () => {
 		return;
 	}
 
-	await lockManager.acquireLock("classifyVideos", 5 * 60);
+	await lockManager.acquireLock("classifyVideos");
 
 	const videos = await getUnlabelledVideos(sql);
 	logger.log(`Found ${videos.length} unlabelled videos`);
 
-	const startTime = new Date().getTime();
+	let i = 0;
 	for (const aid of videos) {
-		const now = new Date().getTime();
+		if (i > 200) {
-		if (now - startTime > 4.2 * MINUTE) {
 			await lockManager.releaseLock("classifyVideos");
-			return 1;
+			return 10000 + i;
 		}
 		await ClassifyVideoQueue.add("classifyVideo", { aid: Number(aid) });
+		i++;
 	}
 	await lockManager.releaseLock("classifyVideos");
 	return 0;

packages/crawler/mq/exec/collectSongs.ts

@@ -4,4 +4,4 @@ import { collectSongs } from "mq/task/collectSongs.ts";
 export const collectSongsWorker = async (_job: Job): Promise<void> =>{
 	await collectSongs();
 	return;
-};
+}

packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts

@@ -16,8 +16,8 @@ export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {
 			if (eta > 144) continue;
 			const now = Date.now();
 			const scheduledNextSnapshotDelay = eta * HOUR;
-			const maxInterval = 1.2 * HOUR;
+			const maxInterval = 1 * HOUR;
-			const minInterval = 2 * SECOND;
+			const minInterval = 1 * SECOND;
 			const delay = truncate(scheduledNextSnapshotDelay, minInterval, maxInterval);
 			const targetTime = now + delay;
 			await scheduleSnapshot(sql, aid, "milestone", targetTime);
@@ -25,5 +25,5 @@ export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {
 		}
 	} catch (e) {
 		logger.error(e as Error, "mq", "fn:dispatchMilestoneSnapshotsWorker");
-	}
 	};
+}

packages/crawler/mq/exec/dispatchRegularSnapshots.ts

@@ -1,7 +1,7 @@
 import { Job } from "bullmq";
 import { getLatestVideoSnapshot } from "db/snapshot.ts";
 import { truncate } from "utils/truncate.ts";
-import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
+import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { HOUR, MINUTE, WEEK } from "@core/const/time.ts";
 import { lockManager } from "@core/mq/lockManager.ts";
@@ -17,7 +17,7 @@ export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise<void> =
 		}
 		await lockManager.acquireLock("dispatchRegularSnapshots", 30 * 60);
 
-		const aids = await getVideosWithoutActiveSnapshotScheduleByType(sql, "normal");
+		const aids = await getVideosWithoutActiveSnapshotSchedule(sql);
 		for (const rawAid of aids) {
 			const aid = Number(rawAid);
 			const latestSnapshot = await getLatestVideoSnapshot(sql, aid);

packages/crawler/mq/exec/getLatestVideos.ts

@@ -4,4 +4,4 @@ import { queueLatestVideos } from "mq/task/queueLatestVideo.ts";
 
 export const getLatestVideosWorker = async (_job: Job): Promise<void> =>{
 	await queueLatestVideos(sql);
-};
+}

packages/crawler/mq/exec/getVideoInfo.ts

@@ -10,4 +10,4 @@ export const getVideoInfoWorker = async (job: Job): Promise<void> => {
 		return;
 	}
 	await insertVideoInfo(sql, aid);
-};
+}

packages/crawler/mq/exec/snapshotTick.ts

@@ -5,15 +5,15 @@ import {
 	getBulkSnapshotsInNextSecond,
 	getSnapshotsInNextSecond,
 	setSnapshotStatus,
-	videoHasProcessingSchedule
+	videoHasProcessingSchedule,
 } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { SnapshotQueue } from "mq/index.ts";
 import { sql } from "@core/db/dbNew";
 
 const priorityMap: { [key: string]: number } = {
-	milestone: 1,
+	"milestone": 1,
-	normal: 3
+	"normal": 3,
 };
 
 export const bulkSnapshotTickWorker = async (_job: Job) => {
@@ -35,16 +35,12 @@ export const bulkSnapshotTickWorker = async (_job: Job) => {
 					created_at: schedule.created_at,
 					started_at: schedule.started_at,
 					finished_at: schedule.finished_at,
-					status: schedule.status
+					status: schedule.status,
 				};
 			});
-			await SnapshotQueue.add(
+			await SnapshotQueue.add("bulkSnapshotVideo", {
-				"bulkSnapshotVideo",
+				schedules: schedulesData,
-				{
+			}, { priority: 3 });
-					schedules: schedulesData
-				},
-				{ priority: 3 }
-			);
 		}
 		return `OK`;
 	} catch (e) {
@@ -65,15 +61,11 @@ export const snapshotTickWorker = async (_job: Job) => {
 			}
 			const aid = Number(schedule.aid);
 			await setSnapshotStatus(sql, schedule.id, "processing");
-			await SnapshotQueue.add(
+			await SnapshotQueue.add("snapshotVideo", {
-				"snapshotVideo",
-				{
 				aid: Number(aid),
 				id: Number(schedule.id),
-					type: schedule.type ?? "normal"
+				type: schedule.type ?? "normal",
-				},
+			}, { priority });
-				{ priority }
-			);
 		}
 		return `OK`;
 	} catch (e) {
@@ -84,5 +76,5 @@ export const snapshotTickWorker = async (_job: Job) => {
 export const closetMilestone = (views: number) => {
 	if (views < 100000) return 100000;
 	if (views < 1000000) return 1000000;
-	return Math.ceil(views / 1000000) * 1000000;
+	return 10000000;
 };

packages/crawler/mq/exec/snapshotVideo.ts

@@ -1,19 +1,19 @@
 import { Job } from "bullmq";
-import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
+import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
+import { lockManager } from "@core/mq/lockManager.ts";
 import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
 import { insertVideoSnapshot } from "mq/task/getVideoStats.ts";
 import { getSongsPublihsedAt } from "db/songs.ts";
 import { getAdjustedShortTermETA } from "mq/scheduling.ts";
 import { NetSchedulerError } from "@core/net/delegate.ts";
 import { sql } from "@core/db/dbNew.ts";
-import { closetMilestone } from "./snapshotTick.ts";
 
 const snapshotTypeToTaskMap: { [key: string]: string } = {
-	milestone: "snapshotMilestoneVideo",
+	"milestone": "snapshotMilestoneVideo",
-	normal: "snapshotVideo",
+	"normal": "snapshotVideo",
-	new: "snapshotMilestoneVideo"
+	"new": "snapshotMilestoneVideo",
 };
 
 export const snapshotVideoWorker = async (job: Job): Promise<void> => {
@@ -22,7 +22,6 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 	const type = job.data.type;
 	const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo";
 	const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE;
-	const latestSnapshot = await getLatestSnapshot(sql, aid);
 	try {
 		const exists = await snapshotScheduleExists(sql, id);
 		if (!exists) {
@@ -33,7 +32,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 			logger.warn(
 				`Video ${aid} has status ${status} in the database. Abort snapshoting.`,
 				"mq",
-				"fn:dispatchRegularSnapshotsWorker"
+				"fn:dispatchRegularSnapshotsWorker",
 			);
 			return;
 		}
@@ -45,7 +44,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 			logger.warn(
 				`Bilibili return status ${status} when snapshoting for ${aid}.`,
 				"mq",
-				"fn:dispatchRegularSnapshotsWorker"
+				"fn:dispatchRegularSnapshotsWorker",
 			);
 			return;
 		}
@@ -53,7 +52,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 		if (type === "new") {
 			const publihsedAt = await getSongsPublihsedAt(sql, aid);
 			const timeSincePublished = stat.time - publihsedAt!;
-			const viewsPerHour = (stat.views / timeSincePublished) * HOUR;
+			const viewsPerHour = stat.views / timeSincePublished * HOUR;
 			if (timeSincePublished > 48 * HOUR) {
 				return;
 			}
@@ -73,41 +72,46 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 			await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true);
 		}
 		if (type !== "milestone") return;
-		const alreadyAchievedMilestone = stat.views > closetMilestone(latestSnapshot.views);
-		if (alreadyAchievedMilestone) {
-			return;
-		}
 		const eta = await getAdjustedShortTermETA(sql, aid);
 		if (eta > 144) {
 			const etaHoursString = eta.toFixed(2) + " hrs";
 			logger.warn(
 				`ETA (${etaHoursString}) too long for milestone snapshot. aid: ${aid}.`,
 				"mq",
-				"fn:snapshotVideoWorker"
+				"fn:dispatchRegularSnapshotsWorker",
 			);
-			return;
 		}
 		const now = Date.now();
 		const targetTime = now + eta * HOUR;
 		await scheduleSnapshot(sql, aid, type, targetTime);
 		await setSnapshotStatus(sql, id, "completed");
 		return;
-	} catch (e) {
+	}
+	catch (e) {
 		if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
-			logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker");
+			logger.warn(
+				`No available proxy for aid ${job.data.aid}.`,
+				"mq",
+				"fn:takeSnapshotForVideoWorker",
+			);
 			await setSnapshotStatus(sql, id, "no_proxy");
-			await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true);
+			await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval);
 			return;
-		} else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
+		}
+		else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
 			logger.warn(
 				`Failed to proxy request for aid ${job.data.aid}: ${e.message}`,
 				"mq",
-				"fn:snapshotVideoWorker"
+				"fn:takeSnapshotForVideoWorker",
 			);
 			await setSnapshotStatus(sql, id, "failed");
 			await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval);
 		}
-		logger.error(e as Error, "mq", "fn:snapshotVideoWorker");
+		logger.error(e as Error, "mq", "fn:takeSnapshotForVideoWorker");
 		await setSnapshotStatus(sql, id, "failed");
 	}
+	finally {
+		await lockManager.releaseLock("dispatchRegularSnapshots");
+	};
+	return;
 };

packages/crawler/mq/exec/takeBulkSnapshot.ts

@@ -3,7 +3,7 @@ import {
 	bulkScheduleSnapshot,
 	bulkSetSnapshotStatus,
 	scheduleSnapshot,
-	snapshotScheduleExists
+	snapshotScheduleExists,
 } from "db/snapshotSchedule.ts";
 import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts";
 import logger from "@core/log/logger.ts";
@@ -55,7 +55,7 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
 					${shares}, 
 					${favorites}
 				)
-			`;
+			`
 
 			logger.log(`Taken snapshot for video ${aid} in bulk.`, "net", "fn:takeBulkSnapshotForVideosWorker");
 		}
@@ -72,16 +72,13 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
 		return `DONE`;
 	} catch (e) {
 		if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
-			logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker");
+			logger.warn(
-			await bulkSetSnapshotStatus(sql, ids, "no_proxy");
+				`No available proxy for bulk request now.`,
-			await bulkScheduleSnapshot(
+				"mq",
-				sql,
+				"fn:takeBulkSnapshotForVideosWorker",
-				aidsToFetch,
-				"normal",
-				Date.now() + 20 * MINUTE * Math.random(),
-				false,
-				true
 			);
+			await bulkSetSnapshotStatus(sql, ids, "no_proxy");
+			await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random());
 			return;
 		}
 		logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");

packages/crawler/mq/init.ts

@@ -62,8 +62,8 @@ export async function initMQ() {
 	});
 
 	await SnapshotQueue.upsertJobScheduler("dispatchArchiveSnapshots", {
-		every: 2 * HOUR,
+		every: 6 * HOUR,
-		immediately: false
+		immediately: true
 	});
 
 	await SnapshotQueue.upsertJobScheduler("scheduleCleanup", {

packages/crawler/mq/scheduling.ts

@@ -33,7 +33,7 @@ export const getAdjustedShortTermETA = async (sql: Psql, aid: number) => {
 	if (!snapshotsEnough) return 0;
 
 	const currentTimestamp = new Date().getTime();
-	const timeIntervals = [3 * MINUTE, 20 * MINUTE, HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
+	const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
 	const DELTA = 0.00001;
 	let minETAHours = Infinity;
 

packages/crawler/mq/task/collectSongs.ts

@@ -25,5 +25,5 @@ export async function insertIntoSongs(sql: Psql, aid: number) {
 			(SELECT duration FROM bilibili_metadata WHERE aid = ${aid})
 		)
 		ON CONFLICT DO NOTHING
-	`;
+	`
 }

packages/crawler/mq/task/getVideoDetails.ts

@@ -18,9 +18,9 @@ export async function insertVideoInfo(sql: Psql, aid: number) {
 	const bvid = data.View.bvid;
 	const desc = data.View.desc;
 	const uid = data.View.owner.mid;
-	const tags = data.Tags.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type))
+	const tags = data.Tags
-		.map((tag) => tag.tag_name)
+		.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type))
-		.join(",");
+		.map((tag) => tag.tag_name).join(",");
 	const title = data.View.title;
 	const published_at = formatTimestampToPsql(data.View.pubdate * SECOND + 8 * HOUR);
 	const duration = data.View.duration;
@@ -55,7 +55,7 @@ export async function insertVideoInfo(sql: Psql, aid: number) {
 			${stat.share}, 
 			${stat.favorite}
 		)
-	`;
+	`
 
 	logger.log(`Inserted video metadata for aid: ${aid}`, "mq");
 	await ClassifyVideoQueue.add("classifyVideo", { aid });

packages/crawler/mq/task/getVideoStats.ts

@@ -24,7 +24,11 @@ export interface SnapshotNumber {
  * - The native `fetch` function threw an error: with error code `FETCH_ERROR`
  * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
  */
-export async function insertVideoSnapshot(sql: Psql, aid: number, task: string): Promise<number | SnapshotNumber> {
+export async function insertVideoSnapshot(
+	sql: Psql,
+	aid: number,
+	task: string,
+): Promise<number | SnapshotNumber> {
 	const data = await getVideoInfo(aid, task);
 	if (typeof data == "number") {
 		return data;
@@ -41,7 +45,7 @@ export async function insertVideoSnapshot(sql: Psql, aid: number, task: string):
     await sql`
         INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
         VALUES (${aid}, ${views}, ${danmakus}, ${replies}, ${likes}, ${coins}, ${shares}, ${favorites})
-    `;
+    `
 
 	logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
 
@@ -54,6 +58,6 @@ export async function insertVideoSnapshot(sql: Psql, aid: number, task: string):
 		coins,
 		shares,
 		favorites,
-		time
+		time,
 	};
 }

packages/crawler/mq/task/queueLatestVideo.ts

@@ -6,7 +6,9 @@ import logger from "@core/log/logger.ts";
 import { LatestVideosQueue } from "mq/index.ts";
 import type { Psql } from "@core/db/psql.d.ts";
 
-export async function queueLatestVideos(sql: Psql): Promise<number | null> {
+export async function queueLatestVideos(
+	sql: Psql,
+): Promise<number | null> {
 	let page = 1;
 	let i = 0;
 	const videosFound = new Set();
@@ -24,18 +26,14 @@ export async function queueLatestVideos(sql: Psql): Promise<number | null> {
 			if (videoExists) {
 				continue;
 			}
-			await LatestVideosQueue.add(
+			await LatestVideosQueue.add("getVideoInfo", { aid }, {
-				"getVideoInfo",
-				{ aid },
-				{
 				delay,
 				attempts: 100,
 				backoff: {
 					type: "fixed",
-						delay: SECOND * 5
+					delay: SECOND * 5,
-					}
+				},
-				}
+			});
-			);
 			videosFound.add(aid);
 			allExists = false;
 			delay += Math.random() * SECOND * 1.5;
@@ -44,7 +42,7 @@ export async function queueLatestVideos(sql: Psql): Promise<number | null> {
 		logger.log(
 			`Page ${page} crawled, total: ${videosFound.size}/${i} videos added/observed.`,
 			"net",
-			"fn:queueLatestVideos()"
+			"fn:queueLatestVideos()",
 		);
 		if (allExists) {
 			return 0;

packages/crawler/mq/task/regularSnapshotInterval.ts

@@ -1,6 +1,6 @@
 import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts";
 import { HOUR } from "@core/const/time.ts";
-import type { Psql } from "@core/db/psql.d.ts";
+import type { Psql } from "@core/db/psql";
 
 export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => {
 	const now = Date.now();

packages/crawler/mq/task/removeAllTimeoutSchedules.ts

@@ -2,7 +2,11 @@ import { sql } from "@core/db/dbNew";
 import logger from "@core/log/logger.ts";
 
 export async function removeAllTimeoutSchedules() {
-	logger.log("Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker");
+	logger.log(
+		"Too many timeout schedules, directly removing these schedules...",
+		"mq",
+		"fn:scheduleCleanupWorker",
+	);
 	return await sql`
 		DELETE FROM snapshot_schedule
 		WHERE status IN ('pending', 'processing')

packages/crawler/package.json

@@ -1,6 +1,5 @@
 {
 	"name": "crawler",
-	"version": "1.3.0",
 	"scripts": {
 		"test": "bun --env-file=.env.test run vitest",
 		"worker:main": "bun run ./src/worker.ts",
@@ -8,8 +7,7 @@
 		"worker:filter": "bun run ./build/filterWorker.js",
 		"adder": "bun run ./src/jobAdder.ts",
 		"bullui": "bun run ./src/bullui.ts",
-		"all": "bun run concurrently --restart-tries -1 'bun run worker:main' 'bun run adder' 'bun run worker:filter'",
+		"all": "bun run concurrently --restart-tries -1 'bun run worker:main' 'bun run adder' 'bun run bullui' 'bun run worker:filter'"
-		"format": "prettier --write ."
 	},
 	"devDependencies": {
 		"concurrently": "^9.1.2"

packages/crawler/src/build.ts

@@ -6,6 +6,7 @@ await Bun.build({
     target: "node"
 });
 
+
 const file = Bun.file("./build/filterWorker.js");
 const code = await file.text();
 

packages/crawler/src/bullui.ts

@@ -11,9 +11,9 @@ createBullBoard({
 	queues: [
 		new BullMQAdapter(LatestVideosQueue),
 		new BullMQAdapter(ClassifyVideoQueue),
-		new BullMQAdapter(SnapshotQueue)
+		new BullMQAdapter(SnapshotQueue),
 	],
-	serverAdapter: serverAdapter
+	serverAdapter: serverAdapter,
 });
 
 const app = express();

packages/crawler/src/filterWorker.ts

@@ -12,8 +12,8 @@ const shutdown = async (signal: string) => {
     process.exit(0);
 };
 
-process.on("SIGINT", () => shutdown("SIGINT"));
+process.on('SIGINT', () => shutdown('SIGINT'));
-process.on("SIGTERM", () => shutdown("SIGTERM"));
+process.on('SIGTERM', () => shutdown('SIGTERM'));
 
 await Akari.init();
 
@@ -29,7 +29,7 @@ const filterWorker = new Worker(
 				break;
 		}
 	},
-	{ connection: redis as ConnectionOptions, concurrency: 2, removeOnComplete: { count: 1000 } }
+	{ connection: redis as ConnectionOptions, concurrency: 2, removeOnComplete: { count: 1000 } },
 );
 
 filterWorker.on("active", () => {

packages/crawler/src/worker.ts

@@ -10,7 +10,7 @@ import {
 	scheduleCleanupWorker,
 	snapshotTickWorker,
 	snapshotVideoWorker,
-	takeBulkSnapshotForVideosWorker
+	takeBulkSnapshotForVideosWorker,
 } from "mq/exec/executors.ts";
 import { redis } from "@core/db/redis.ts";
 import logger from "@core/log/logger.ts";
@@ -37,8 +37,8 @@ const shutdown = async (signal: string) => {
     process.exit(0);
 };
 
-process.on("SIGINT", () => shutdown("SIGINT"));
+process.on('SIGINT', () => shutdown('SIGINT'));
-process.on("SIGTERM", () => shutdown("SIGTERM"));
+process.on('SIGTERM', () => shutdown('SIGTERM'));
 
 const latestVideoWorker = new Worker(
 	"latestVideos",
@@ -58,8 +58,8 @@ const latestVideoWorker = new Worker(
 		connection: redis as ConnectionOptions,
 		concurrency: 6,
 		removeOnComplete: { count: 1440 },
-		removeOnFail: { count: 0 }
+		removeOnFail: { count: 0 },
-	}
+	},
 );
 
 latestVideoWorker.on("active", () => {
@@ -95,7 +95,7 @@ const snapshotWorker = new Worker(
 				break;
 		}
 	},
-	{ connection: redis as ConnectionOptions, concurrency: 50, removeOnComplete: { count: 2000 } }
+	{ connection: redis as ConnectionOptions, concurrency: 50, removeOnComplete: { count: 2000 } },
 );
 
 snapshotWorker.on("error", (err) => {

packages/crawler/test/db/snapshotSchedule.test.ts

@@ -56,26 +56,26 @@ const databasePreparationQuery = `
     CREATE INDEX idx_snapshot_schedule_status ON snapshot_schedule USING btree (status);
     CREATE INDEX idx_snapshot_schedule_type ON snapshot_schedule USING btree (type);
     CREATE UNIQUE INDEX snapshot_schedule_pkey ON snapshot_schedule USING btree (id);
-`;
+`
 
 const cleanUpQuery = `
     DROP SEQUENCE IF EXISTS "snapshot_schedule_id_seq" CASCADE;
     DROP TABLE IF EXISTS "snapshot_schedule" CASCADE;
-`;
+`
 
 async function testMocking() {
-	await sql.begin(async (tx) => {
+	await sql.begin(async tx => {
         await tx.unsafe(cleanUpQuery).simple();
 		await tx.unsafe(databasePreparationQuery).simple();
         
         await tx`
             INSERT INTO snapshot_schedule 
-            ${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
+            ${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')}
         `;
 
         await tx`
             ROLLBACK;
-        `;
+        `
 
         await tx.unsafe(cleanUpQuery).simple();
         return;
@@ -83,18 +83,18 @@ async function testMocking() {
 }
 
 async function testBulkSetSnapshotStatus() {
-	return await sql.begin(async (tx) => {
+    return await sql.begin(async tx => {
         await tx.unsafe(cleanUpQuery).simple();
         await tx.unsafe(databasePreparationQuery).simple();
 
         await tx`
             INSERT INTO snapshot_schedule 
-            ${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
+            ${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')}
         `;
 
 		const ids = [1, 2, 3];
 		
-		await bulkSetSnapshotStatus(tx, ids, "pending");
+        await bulkSetSnapshotStatus(tx, ids, 'pending')
 
         const rows = tx<{status: string}[]>`
             SELECT status FROM snapshot_schedule WHERE id = 1;
@@ -102,7 +102,7 @@ async function testBulkSetSnapshotStatus() {
 
         await tx`
             ROLLBACK;
-        `;
+        `
 
         await tx.unsafe(cleanUpQuery).simple();
         return rows;
@@ -116,5 +116,5 @@ test("data mocking works", async () => {
 
 test("bulkSetSnapshotStatus core logic works smoothly", async () => {
     const rows = await testBulkSetSnapshotStatus();
-	expect(rows.every((item) => item.status === "pending")).toBe(true);
+	expect(rows.every(item => item.status === 'pending')).toBe(true);
 });

packages/crawler/utils/formatTimestampToPostgre.ts

@@ -1,6 +1,6 @@
 export function formatTimestampToPsql(timestamp: number) {
 	const date = new Date(timestamp);
-	return date.toISOString().slice(0, 23).replace("T", " ") + "+08";
+	return date.toISOString().slice(0, 23).replace("T", " ");
 }
 
 export function parseTimestampFromPsql(timestamp: string) {

packages/crawler/vitest.config.ts

@@ -1,5 +1,5 @@
 import { defineConfig } from "vitest/config";
-import tsconfigPaths from "vite-tsconfig-paths";
+import tsconfigPaths from 'vite-tsconfig-paths'
 
 export default defineConfig({
     plugins: [tsconfigPaths()]

packages/frontend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "frontend",
   "type": "module",
-	"version": "1.9.0",
+  "version": "0.0.1",
   "scripts": {
     "dev": "astro dev",
     "build": "astro build",

packages/next/app/[locale]/login/page.tsx

@@ -1,20 +1,9 @@
 import { LeftArrow } from "@/components/icons/LeftArrow";
 import { RightArrow } from "@/components/icons/RightArrow";
 import LoginForm from "./LoginForm";
-import { Link, redirect } from "@/i18n/navigation";
+import { Link } from "@/i18n/navigation";
-import { getLocale } from "next-intl/server";
-import { getCurrentUser } from "@/lib/userAuth";
 
-export default async function LoginPage() {
+export default function SignupPage() {
-	const user = await getCurrentUser();
-	const locale = await getLocale();
-
-	if (user) {
-		redirect({
-			href: `/user/${user.uid}/profile`,
-			locale: locale
-		});
-	}
 	return (
 		<main className="relative flex-grow pt-8 px-4 md:w-full md:h-full md:flex md:items-center md:justify-center">
 			<div

packages/next/app/[locale]/logout/route.ts

@@ -1,60 +0,0 @@
-import { ApiRequestError, fetcher } from "@/lib/net";
-import { ErrorResponse } from "@cvsa/backend";
-import { cookies } from "next/headers";
-
-export async function POST() {
-	const backendURL = process.env.BACKEND_URL || "";
-	const cookieStore = await cookies();
-	const sessionID = cookieStore.get("session_id");
-	if (!sessionID) {
-		const response: ErrorResponse<string> = {
-			message: "No session_id provided",
-			errors: [],
-			code: "ENTITY_NOT_FOUND"
-		};
-		return new Response(JSON.stringify(response), {
-			status: 401
-		});
-	}
-
-	try {
-		const response = await fetcher(`${backendURL}/session/${sessionID.value}`, {
-			method: "DELETE"
-		});
-
-		const headers = response.headers;
-
-		return new Response(null, {
-			status: 204,
-			headers: {
-				"Set-Cookie": (headers["set-cookie"] || [""])[0]
-			}
-		});
-	} catch (error) {
-		if (error instanceof ApiRequestError && error.response) {
-			const res = error.response;
-			const code = error.code;
-			return new Response(JSON.stringify(res), {
-				status: code
-			});
-		} else if (error instanceof Error) {
-			const response: ErrorResponse<string> = {
-				message: error.message,
-				errors: [],
-				code: "SERVER_ERROR"
-			};
-			return new Response(JSON.stringify(response), {
-				status: 500
-			});
-		} else {
-			const response: ErrorResponse<string> = {
-				message: "Unknown error occurred",
-				errors: [],
-				code: "UNKNOWN_ERROR"
-			};
-			return new Response(JSON.stringify(response), {
-				status: 500
-			});
-		}
-	}
-}

packages/next/app/[locale]/my/profile/page.tsx

@@ -0,0 +1,46 @@
+import { Header } from "@/components/shell/Header";
+import { getCurrentUser, User } from "@/lib/userAuth";
+import { redirect } from "next/navigation";
+import { format } from "date-fns";
+import { zhCN } from "date-fns/locale";
+
+interface SignupTimeProps {
+	user: User;
+}
+
+const SignupTime: React.FC<SignupTimeProps> = ({ user }: SignupTimeProps) => {
+	return (
+		<p className="mt-4">
+			于&nbsp;
+			{format(new Date(user.createdAt), "yyyy-MM-dd HH:mm:ss", {
+				locale: zhCN
+			})}
+			&nbsp;注册。
+		</p>
+	);
+};
+
+export default async function ProfilePage() {
+	const user = await getCurrentUser();
+
+	if (!user) {
+		redirect("/login");
+	}
+
+	const displayName = user.nickname || user.username;
+
+	return (
+		<>
+			<Header user={user} />
+			<main className="md:w-xl lg:w-2xl xl:w-3xl md:mx-auto pt-6">
+				<h1>
+					<span className="text-4xl font-extralight">{displayName}</span>
+					<span className="ml-2 text-on-surface-variant dark:text-dark-on-surface-variant">
+						UID{user.uid}
+					</span>
+				</h1>
+				<SignupTime user={user} />
+			</main>
+		</>
+	);
+}

packages/next/app/[locale]/song/[id]/info/page.tsx

@@ -7,7 +7,6 @@ import { aidExists as idExists } from "@/lib/db/bilibili_metadata/aidExists";
 import { notFound } from "next/navigation";
 import { BiliVideoMetadataType, VideoSnapshotType } from "@cvsa/core";
 import { Metadata } from "next";
-import { DateTime } from "luxon";
 
 const MetadataRow = ({ title, desc }: { title: string; desc: string | number | undefined | null }) => {
 	if (!desc) return <></>;
@@ -105,16 +104,18 @@ export default async function VideoInfoPage({ params }: { params: Promise<{ id:
 									title="发布时间"
 									desc={
 										videoInfo.published_at
-											? DateTime.fromJSDate(videoInfo.published_at).toFormat(
+											? format(new Date(videoInfo.published_at), "yyyy-MM-dd HH:mm:ss", {
-													"yyyy-MM-dd HH:mm:ss"
+													locale: zhCN
-												)
+												})
 											: null
 									}
 								/>
 								<MetadataRow title="时长 (秒)" desc={videoInfo.duration} />
 								<MetadataRow
 									title="创建时间"
-									desc={DateTime.fromJSDate(videoInfo.created_at).toFormat("yyyy-MM-dd HH:mm:ss")}
+									desc={format(new Date(videoInfo.created_at), "yyyy-MM-dd HH:mm:ss", {
+										locale: zhCN
+									})}
 								/>
 								<MetadataRow title="封面" desc={videoInfo?.cover_url} />
 							</tbody>
@@ -141,11 +142,11 @@ export default async function VideoInfoPage({ params }: { params: Promise<{ id:
 								</thead>
 								<tbody>
 									{snapshots.map((snapshot) => (
-										<tr key={snapshot.id}>
+										<tr key={snapshot.created_at}>
 											<td className="border dark:border-zinc-500 px-4 py-2">
-												{DateTime.fromJSDate(snapshot.created_at).toFormat(
+												{format(new Date(snapshot.created_at), "yyyy-MM-dd HH:mm:ss", {
-													"yyyy-MM-dd HH:mm:ss"
+													locale: zhCN
-												)}
+												})}
 											</td>
 											<td className="border dark:border-zinc-500 px-4 py-2">{snapshot.views}</td>
 											<td className="border dark:border-zinc-500 px-4 py-2">{snapshot.coins}</td>

packages/next/app/[locale]/song/layout.tsx

@@ -1,4 +1,5 @@
-import HeaderServer from "@/components/shell/HeaderServer";
+import { Header } from "@/components/shell/Header";
+import { getCurrentUser } from "@/lib/userAuth";
 import React from "react";
 
 export default async function RootLayout({
@@ -6,9 +7,10 @@ export default async function RootLayout({
 }: Readonly<{
 	children: React.ReactNode;
 }>) {
+	const user = await getCurrentUser();
 	return (
 		<>
-			<HeaderServer />
+			<Header user={user} />
 			{children}
 		</>
 	);

packages/next/app/[locale]/test-vdf/page.tsx

@@ -1,10 +1,13 @@
+import { Header } from "@/components/shell/Header";
+import { getCurrentUser } from "@/lib/userAuth";
 import { VDFtestCard } from "./TestCard";
-import HeaderServer from "@/components/shell/HeaderServer";
 
 export default async function VdfBenchmarkPage() {
+	const user = await getCurrentUser();
+
 	return (
 		<>
-			<HeaderServer />
+			<Header user={user} />
 			<div className="md:w-2/3 lg:w-1/2 xl:w-[37%] md:mx-auto mx-6 mb-12">
 				<VDFtestCard />
 				<div>

packages/next/app/[locale]/user/[uid]/profile/LogoutButton.tsx

@@ -1,46 +0,0 @@
-"use client";
-
-import { FilledButton } from "@/components/ui/Buttons/FilledButton";
-import { Dialog, DialogButton, DialogButtonGroup, DialogHeadline, DialogSupportingText } from "@/components/ui/Dialog";
-import { Portal } from "@/components/utils/Portal";
-import { useRouter } from "@/i18n/navigation";
-import { useState } from "react";
-
-export const LogoutButton: React.FC = () => {
-	const [showDialog, setShowDialog] = useState(false);
-	const router = useRouter();
-	return (
-		<>
-			<FilledButton
-				shape="square"
-				className="mt-5 !text-on-error dark:!text-dark-on-error !bg-error dark:!bg-dark-error font-medium"
-				onClick={() => setShowDialog(true)}
-			>
-				登出
-			</FilledButton>
-			<Portal>
-				<Dialog show={showDialog}>
-					<DialogHeadline>确认登出</DialogHeadline>
-					<DialogSupportingText>确认要退出登录吗？</DialogSupportingText>
-					<DialogButtonGroup close={() => setShowDialog(false)}>
-						<DialogButton onClick={() => setShowDialog(false)}>取消</DialogButton>
-						<DialogButton
-							onClick={async () => {
-								try {
-									await fetch("/logout", {
-										method: "POST"
-									});
-									router.push("/");
-								} finally {
-									setShowDialog(false);
-								}
-							}}
-						>
-							确认
-						</DialogButton>
-					</DialogButtonGroup>
-				</Dialog>
-			</Portal>
-		</>
-	);
-};

packages/next/app/[locale]/user/[uid]/profile/page.tsx

@@ -1,64 +0,0 @@
-import { getUserProfile, User } from "@/lib/userAuth";
-import { notFound } from "next/navigation";
-import { format } from "date-fns";
-import { zhCN } from "date-fns/locale";
-import { LogoutButton } from "./LogoutButton";
-import { numeric } from "yup-numeric";
-import { getTranslations } from "next-intl/server";
-import HeaderServer from "@/components/shell/HeaderServer";
-import { DateTime } from "luxon";
-
-const uidSchema = numeric().integer().min(0);
-
-interface SignupTimeProps {
-	user: User;
-}
-
-const SignupTime: React.FC<SignupTimeProps> = ({ user }: SignupTimeProps) => {
-	return (
-		<p className="mt-4">
-			于&nbsp;
-			{DateTime.fromJSDate(user.createdAt).toFormat("yyyy-MM-dd HH:mm:ss")}
-			&nbsp;注册。
-		</p>
-	);
-};
-
-export default async function ProfilePage({ params }: { params: Promise<{ uid: string }> }) {
-	const { uid } = await params;
-	const t = await getTranslations("profile_page");
-	let parsedUID: number;
-
-	try {
-		uidSchema.validate(uid);
-		parsedUID = parseInt(uid);
-	} catch (error) {
-		return notFound();
-	}
-
-	const user = await getUserProfile(parsedUID);
-
-	if (!user) {
-		return notFound();
-	}
-
-	const displayName = user.nickname || user.username;
-	const loggedIn = user.isLoggedIn;
-
-	return (
-		<>
-			<HeaderServer />
-			<main className="md:w-xl lg:w-2xl xl:w-3xl md:mx-auto pt-6">
-				<h1>
-					<span className="text-4xl font-extralight">{displayName}</span>
-					<span className="ml-2 text-on-surface-variant dark:text-dark-on-surface-variant">
-						UID{user.uid}
-					</span>
-				</h1>
-				<SignupTime user={user} />
-				<p className="mt-4">权限组：{t(`role.${user.role}`)}</p>
-				{loggedIn && <LogoutButton />}
-			</main>
-		</>
-	);
-}

packages/next/app/[locale]/video/[id]/info/page.tsx

@@ -1,9 +1,9 @@
 import { Suspense } from "react";
 import Link from "next/link";
+import { format } from "date-fns";
 import { notFound } from "next/navigation";
 import { Metadata } from "next";
 import type { VideoInfoData } from "@cvsa/core";
-import { DateTime } from "luxon";
 
 const StatRow = ({ title, description }: { title: string; description?: number }) => {
 	return (
@@ -51,7 +51,7 @@ const VideoInfo = async ({ id }: { id: string }) => {
 					{data.bvid} · av{data.aid}
 				</span>
 				<br />
-				<span>发布于 {DateTime.fromSeconds(data.pubdate).toFormat("yyyy-MM-dd HH:mm:ss")}</span>
+				<span>发布于 {format(new Date(data.pubdate * 1000), "yyyy-MM-dd HH:mm:ss")}</span>
 				<br />
 				<span>播放：{(data.stat?.view ?? 0).toLocaleString()}</span> ·{" "}
 				<span>弹幕：{(data.stat?.danmaku ?? 0).toLocaleString()}</span>

packages/next/app/[locale]/video/layout.tsx

@@ -1,4 +1,5 @@
-import HeaderServer from "@/components/shell/HeaderServer";
+import { Header } from "@/components/shell/Header";
+import { getCurrentUser } from "@/lib/userAuth";
 import React from "react";
 
 export default async function RootLayout({
@@ -6,9 +7,10 @@ export default async function RootLayout({
 }: Readonly<{
 	children: React.ReactNode;
 }>) {
+	const user = await getCurrentUser();
 	return (
 		<>
-			<HeaderServer />
+			<Header user={user} />
 			{children}
 		</>
 	);

packages/next/bun.lock

@@ -5,19 +5,16 @@
       "name": "next",
       "dependencies": {
         "@cvsa/backend": "^0.5.3",
-        "@cvsa/core": "0.0.10",
+        "@cvsa/core": "^0.0.7",
         "@mdx-js/loader": "^3.1.0",
         "@mdx-js/react": "^3.1.0",
         "@next/mdx": "^15.3.3",
-        "@types/luxon": "^3.6.2",
         "@types/mdx": "^2.0.13",
         "axios": "^1.9.0",
         "date-fns": "^4.1.0",
         "framer-motion": "^12.15.0",
         "fumadocs-mdx": "^11.6.6",
         "i18next": "^25.2.1",
-        "jotai": "^2.12.5",
-        "luxon": "^3.6.1",
         "next": "^15.3.3",
         "next-intl": "^4.1.0",
         "raw-loader": "^4.0.2",
@@ -26,7 +23,6 @@
         "swr": "^2.3.3",
         "ua-parser-js": "^2.0.3",
         "yup": "^1.6.1",
-        "yup-numeric": "^0.5.0",
       },
       "devDependencies": {
         "@tailwindcss/postcss": "^4.1.8",
@@ -49,7 +45,7 @@
 
     "@cvsa/backend": ["@cvsa/backend@0.5.3", "", { "dependencies": { "@rabbit-company/argon2id": "^2.1.0", "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", "yup": "^1.6.1", "zod": "^3.24.3" } }, "sha512-RzGjarU2TOzD6/d6qikE4xd/ZqNQl3jOYtgfJg5kbWFuiXnOgEC9QBTi+adzjmaWFrcpuYck6ooWpg4eT3s43g=="],
 
-    "@cvsa/core": ["@cvsa/core@0.0.10", "", { "dependencies": { "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", "postgres": "^3.4.5", "winston": "^3.17.0" } }, "sha512-8gjSNRyLZcybLiFSUZFPc4nJsLQ7YO8lZSAEFJidyUA3a6CbB/UUC4G5jqWyWJ7xDA39w7szqpbVYKX3fb6W3g=="],
+    "@cvsa/core": ["@cvsa/core@0.0.7", "", { "dependencies": { "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", "postgres": "^3.4.5", "winston": "^3.17.0" } }, "sha512-j2Ksg+ZquHqKPew1JZxw0Q9yckFnzdd8y+DnmVT+OW18j+pKcduB9j0qqBywQGHxGuDYVOGLiPlf+IBXfqQWTg=="],
 
     "@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="],
 
@@ -277,8 +273,6 @@
 
     "@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="],
 
-    "@types/luxon": ["@types/luxon@3.6.2", "", {}, "sha512-R/BdP7OxEMc44l2Ex5lSXHoIXTB2JLNa3y2QISIbr58U/YcsffyQrYW//hZSdrfxrjRZj3GcUoxMPGdO8gSYuw=="],
-
     "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="],
 
     "@types/mdx": ["@types/mdx@2.0.13", "", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="],
@@ -373,8 +367,6 @@
 
     "big.js": ["big.js@5.2.2", "", {}, "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ=="],
 
-    "bignumber.js": ["bignumber.js@9.3.0", "", {}, "sha512-EM7aMFTXbptt/wZdMlBv2t8IViwQL+h6SLHosp8Yf0dqJMTnY6iL32opnAB6kAdL0SZPuvcAzFr31o0c/R3/RA=="],
-
     "binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="],
 
     "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
@@ -709,8 +701,6 @@
 
     "jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="],
 
-    "jotai": ["jotai@2.12.5", "", { "peerDependencies": { "@types/react": ">=17.0.0", "react": ">=17.0.0" }, "optionalPeers": ["@types/react", "react"] }, "sha512-G8m32HW3lSmcz/4mbqx0hgJIQ0ekndKWiYP7kWVKi0p6saLXdSoye+FZiOFyonnd7Q482LCzm8sMDl7Ar1NWDw=="],
-
     "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="],
 
     "json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="],
@@ -767,8 +757,6 @@
 
     "lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="],
 
-    "luxon": ["luxon@3.6.1", "", {}, "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="],
-
     "magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="],
 
     "markdown-extensions": ["markdown-extensions@2.0.0", "", {}, "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q=="],
@@ -1225,8 +1213,6 @@
 
     "yup": ["yup@1.6.1", "", { "dependencies": { "property-expr": "^2.0.5", "tiny-case": "^1.0.3", "toposort": "^2.0.2", "type-fest": "^2.19.0" } }, "sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA=="],
 
-    "yup-numeric": ["yup-numeric@0.5.0", "", { "dependencies": { "typescript": "^5.4.2" }, "peerDependencies": { "bignumber.js": "^9.1.2", "yup": "^1.3.3" } }, "sha512-IrkLyIY0jLwtomVArrjV1Sv2YHOC715UdRPA7WfAJ0upARXLtmnmzszlPQeEoUxtSb3E9mrF8DoFgiQcRkxOLA=="],
-
     "zod": ["zod@3.25.46", "", {}, "sha512-IqRxcHEIjqLd4LNS/zKffB3Jzg3NwqJxQQ0Ns7pdrvgGkwQsEBdEQcOHaBVqvvZArShRzI39+aMST3FBGmTrLQ=="],
 
     "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],

packages/next/components/shell/Header.tsx

@@ -15,12 +15,12 @@ import { InfoIcon } from "@/components/icons/InfoIcon";
 import { HomeIcon } from "@/components/icons/HomeIcon";
 import { TextButton } from "@/components/ui/Buttons/TextButton";
 import { Link } from "@/i18n/navigation";
+import type { UserResponse } from "@cvsa/backend";
 import { LoginIcon } from "../icons/LoginIcon";
 import { AccountIcon } from "../icons/AccountIcon";
-import { User } from "@/lib/userAuth";
 
 interface HeaderProps {
-	user: User | null;
+	user: UserResponse | null;
 }
 
 export const HeaderDestop = ({ user }: HeaderProps) => {
@@ -44,7 +44,7 @@ export const HeaderDestop = ({ user }: HeaderProps) => {
     				text-xl font-medium items-center w-[15rem] min-w-[8rem] mr-4 lg:mr-0 lg:w-[305px] justify-end"
 			>
 				{user ? (
-					<Link href={`/user/${user.uid}/profile`}>{user.nickname || user.username}</Link>
+					<Link href="/my/profile">{user.nickname || user.username}</Link>
 				) : (
 					<Link href="/login">登录</Link>
 				)}
@@ -91,7 +91,7 @@ export const HeaderMobile = ({ user }: HeaderProps) => {
 						</Link>
 
 						{user ? (
-							<Link href={`/user/${user.uid}/profile`}>
+							<Link href="/my/profile">
 								<TextButton className="w-full h-14 flex justify-start" size="m">
 									<div className="flex items-center w-72">
 										<AccountIcon className="text-2xl pr-4" />

packages/next/components/ui/Buttons/FilledButton.tsx

@@ -1,5 +1,3 @@
-"use client";
-
 import useRipple from "@/components/utils/useRipple";
 
 interface FilledButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement> {

packages/next/components/ui/Buttons/TextButton.tsx

@@ -1,5 +1,3 @@
-"use client";
-
 import useRipple from "@/components/utils/useRipple";
 
 interface TextButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement> {
@@ -7,16 +5,13 @@ interface TextButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement>
 	shape?: "round" | "square";
 	children?: React.ReactNode;
 	ripple?: boolean;
-	ref?: React.Ref<HTMLButtonElement>;
 }
 
 export const TextButton = ({
 	children,
 	size = "s",
 	shape = "round",
-	className = "",
+	className,
-	disabled,
-	ref,
 	ripple = true,
 	...rest
 }: TextButtonProps) => {
@@ -34,19 +29,12 @@ export const TextButton = ({
 		<button
 			className={`text-primary dark:text-dark-primary duration-150 select-none
 				flex items-center justify-center relative overflow-hidden
-				disabled:text-on-surface/40 disabled:dark:text-dark-on-surface/40
 				${sizeClasses} ${shapeClasses} ${className}`}
 			{...rest}
 			onMouseDown={onMouseDown}
 			onTouchStart={onTouchStart}
-			disabled={disabled}
-			ref={ref}
 		>
-			<div
+			<div className="absolute w-full h-full hover:bg-primary/10 left-0 top-0"></div>
-				className={`absolute w-full h-full enabled:hover:bg-primary/10 enabled:dark:hover:bg-dark-primary/10
-				${disabled && "bg-on-surface/10 dark:bg-dark-on-surface/10"}
-				left-0 top-0`}
-			></div>
 			{children}
 		</button>
 	);

packages/next/components/ui/Dialog.tsx

@@ -1,12 +1,7 @@
 import { motion, AnimatePresence } from "framer-motion";
-import React, { useRef } from "react";
+import React from "react";
 import { TextButton } from "./Buttons/TextButton";
-import { useEffect, useState } from "react";
+import { useEffect } from "react";
-import { atom, useAtom, useAtomValue, useSetAtom } from "jotai";
-import { useKeyboardShortcuts } from "@/components/utils/useKeyboardEvents";
-import { UAParser } from "ua-parser-js";
-
-const focusedButtonAtom = atom(-1);
 
 export const useDisableBodyScroll = (open: boolean) => {
 	useEffect(() => {
@@ -28,14 +23,10 @@ type ButtonElementAttr = React.HTMLAttributes<HTMLButtonElement>;
 
 type DialogHeadlineProps = OptionalChidrenProps<HeadElementAttr>;
 type DialogSupportingTextProps = OptionalChidrenProps<DivElementAttr>;
-type DialogButtonGroupProps = DivElementAttr & {
+type DialogButtonGroupProps = OptionalChidrenProps<DivElementAttr>;
-	children: React.ReactElement<DialogButtonProps> | React.ReactElement<DialogButtonProps>[];
-	close: () => void;
-};
 
 interface DialogButtonProps extends OptionalChidrenProps<ButtonElementAttr> {
 	onClick?: React.MouseEventHandler<HTMLButtonElement>;
-	index?: number;
 }
 interface DialogProps extends OptionalChidrenProps<DivElementAttr> {
 	show: boolean;
@@ -72,180 +63,48 @@ export const DialogSupportingText: React.FC<DialogSupportingTextProps> = ({
 	);
 };
 
-export const DialogButton: React.FC<DialogButtonProps> = ({ children, onClick, index, ...rest }: DialogButtonProps) => {
+export const DialogButton: React.FC<DialogButtonProps> = ({ children, onClick, ...rest }: DialogButtonProps) => {
-	const buttonRef = useRef<HTMLButtonElement>(null);
-	const focusedButton = useAtomValue(focusedButtonAtom);
-
-	useEffect(() => {
-		if (!buttonRef.current) return;
-		if (focusedButton === index) buttonRef.current.focus();
-	}, [focusedButton]);
-
 	return (
-		<TextButton onClick={onClick} {...rest} ref={buttonRef}>
+		<TextButton onClick={onClick} {...rest}>
 			{children}
 		</TextButton>
 	);
 };
 
-export const DialogButtonGroup: React.FC<DialogButtonGroupProps> = ({
+export const DialogButtonGroup: React.FC<DialogButtonGroupProps> = ({ children, ...rest }: DialogButtonGroupProps) => {
-	children,
-	close,
-	...rest
-}: DialogButtonGroupProps) => {
-	const [focusedButton, setFocusedButton] = useAtom(focusedButtonAtom);
-	const count = React.Children.count(children);
-
-	useKeyboardShortcuts([
-		{
-			key: "Tab",
-			callback: () => {
-				setFocusedButton((focusedButton + 1) % count);
-			},
-			preventDefault: true
-		},
-		{
-			key: "Escape",
-			callback: close,
-			preventDefault: true
-		}
-	]);
-
 	return (
 		<div className="flex justify-end gap-2" {...rest}>
-			{React.Children.map(children, (child, index) => {
+			{children}
-				if (React.isValidElement<DialogButtonProps>(child) && child.type === DialogButton) {
-					return React.cloneElement(child, {
-						index: index
-					});
-				}
-				return child;
-			})}
 		</div>
 	);
 };
 
-const useCompabilityCheck = () => {
-	const [supported, setSupported] = useState(false);
-
-	useEffect(() => {
-		const parser = new UAParser(navigator.userAgent);
-		const result = parser.getResult();
-
-		const { name: browserName, version: browserVersion } = result.browser;
-
-		let isSupported = false;
-
-		if (!browserVersion) {
-			return;
-		}
-		const [major] = browserVersion.split(".").map(Number);
-
-		switch (browserName) {
-			case "Chromium":
-				isSupported = major >= 107;
-				break;
-			case "Firefox":
-				isSupported = major >= 66;
-				break;
-			case "Safari":
-				isSupported = major >= 16;
-				break;
-			default:
-				isSupported = false;
-				break;
-		}
-
-		setSupported(isSupported);
-	}, []);
-
-	return supported;
-};
-
 export const Dialog: React.FC<DialogProps> = ({ show, children, className }: DialogProps) => {
-	const dialogRef = useRef<HTMLDivElement>(null);
-	const contentRef = useRef<HTMLDivElement>(null);
-	const setFocusedButton = useSetAtom(focusedButtonAtom);
-	const isSupported = useCompabilityCheck();
-
-	useEffect(() => {
-		if (!contentRef.current || !dialogRef.current) return;
-
-		const contentHeight = contentRef.current.offsetHeight;
-		const halfSize = (contentHeight + 48) / 2;
-		dialogRef.current.style.top = `calc(50% - ${halfSize}px)`;
-
-		if (!isSupported) {
-			return;
-		}
-
-		dialogRef.current.style.transition = "grid-template-rows cubic-bezier(0.05, 0.7, 0.1, 1.0) 0.35s";
-
-		if (show) {
-			dialogRef.current.style.gridTemplateRows = "1fr";
-		} else {
-			dialogRef.current.style.gridTemplateRows = "0.6fr";
-		}
-	}, [show]);
-
-	useEffect(() => {
-		setFocusedButton(-1);
-	}, [show]);
-
 	useDisableBodyScroll(show);
-
 	return (
 		<AnimatePresence>
 			{show && (
-				<div className="w-full h-full top-0 left-0 absolute flex justify-center">
+				<div className="w-full h-full top-0 left-0 absolute flex items-center justify-center">
 					<motion.div
 						className="fixed top-0 left-0 w-full h-full z-40 bg-black/20 pointer-none"
 						aria-hidden="true"
 						initial={{ opacity: 0 }}
 						animate={{ opacity: 1 }}
 						exit={{ opacity: 0 }}
-						transition={{ duration: 0.35 }}
+						transition={{ duration: 0.3 }}
 					/>
 					<motion.div
 						className={`fixed min-w-[17.5rem] sm:max-w-[35rem] h-auto z-50 bg-surface-container-high
-				            shadow-2xl shadow-shadow/15 rounded-[1.75rem] p-6 dark:bg-dark-surface-container-high mx-2
+				            shadow-2xl shadow-shadow/15 rounded-[1.75rem] p-6 dark:bg-dark-surface-container-high mx-2 ${className}`}
-							origin-top ${className} overflow-hidden grid ${isSupported && "grid-rows-[0fr]"}`}
+						initial={{ opacity: 0.5, transform: "scale(1.1)" }}
-						initial={{
+						animate={{ opacity: 1, transform: "scale(1)" }}
-							opacity: 0,
+						exit={{ opacity: 0 }}
-							transform: "translateY(-24px)",
+						transition={{ ease: [0.31, 0.69, 0.3, 1.02], duration: 0.3 }}
-							gridTemplateRows: isSupported ? undefined : "0fr"
-						}}
-						animate={{
-							opacity: 1,
-							transform: "translateY(0px)",
-							gridTemplateRows: isSupported ? undefined : "1fr"
-						}}
-						exit={{
-							opacity: 0,
-							transform: "translateY(-24px)",
-							gridTemplateRows: isSupported ? undefined : "0fr"
-						}}
-						transition={{ ease: [0.05, 0.7, 0.1, 1.0], duration: 0.35 }}
 						aria-modal="true"
-						ref={dialogRef}
-					>
-						<div className="min-h-0">
-							<motion.div
-								className="origin-top"
-								initial={{ opacity: 0, transform: "translateY(5px)" }}
-								animate={{ opacity: 1, transform: "translateY(0px)" }}
-								exit={{ opacity: 0, transform: "translateY(5px)" }}
-								transition={{
-									ease: [0.05, 0.7, 0.1, 1.0],
-									duration: 0.35
-								}}
-								ref={contentRef}
 					>
 						{children}
 					</motion.div>
 				</div>
-					</motion.div>
-				</div>
 			)}
 		</AnimatePresence>
 	);

packages/next/components/ui/SearchBox.tsx

@@ -58,7 +58,7 @@ export const SearchBox: React.FC<SearchBoxProps> = ({ close = () => {} }) => {
 						type="search"
 						placeholder="搜索"
 						autoComplete="off"
-						autoCapitalize="none"
+						autoCapitalize="off"
 						autoCorrect="off"
 						className="bg-transparent h-full w-full focus:outline-none"
 						onKeyDown={handleKeyDown}
@@ -73,7 +73,7 @@ export const SearchBox: React.FC<SearchBoxProps> = ({ close = () => {} }) => {
 						type="search"
 						placeholder="搜索"
 						autoComplete="off"
-						autoCapitalize="none"
+						autoCapitalize="off"
 						autoCorrect="off"
 						className="bg-transparent h-full w-full focus:outline-none"
 						onKeyDown={handleKeyDown}

packages/next/components/utils/ErrorDialog.tsx

@@ -14,7 +14,7 @@ export const ErrorDialog: React.FC<ErrorDialogProps> = ({ children, closeDialog,
 		<>
 			<DialogHeadline>{errorCode ? t(errorCode) : "错误"}</DialogHeadline>
 			<DialogSupportingText>{children}</DialogSupportingText>
-			<DialogButtonGroup close={closeDialog}>
+			<DialogButtonGroup>
 				<DialogButton onClick={closeDialog}>关闭</DialogButton>
 			</DialogButtonGroup>
 		</>

packages/next/components/utils/useKeyboardEvents.tsx

@@ -1,31 +0,0 @@
-import { useEffect, useCallback } from "react";
-
-export type KeyboardShortcut = {
-	key: string;
-	callback: () => void;
-	preventDefault?: boolean;
-};
-
-export function useKeyboardShortcuts(shortcuts: KeyboardShortcut[]): void {
-	const handleKeyDown = useCallback(
-		(event: KeyboardEvent) => {
-			shortcuts.forEach((shortcut) => {
-				if (event.key === shortcut.key) {
-					if (shortcut.preventDefault) {
-						event.preventDefault();
-					}
-					shortcut.callback();
-				}
-			});
-		},
-		[shortcuts]
-	);
-
-	useEffect(() => {
-		document.addEventListener("keydown", handleKeyDown);
-
-		return () => {
-			document.removeEventListener("keydown", handleKeyDown);
-		};
-	}, [handleKeyDown]);
-}

packages/next/i18n/strings/zh.json

@@ -7,13 +7,6 @@
 	"username": "用户名",
 	"password": "密码",
 	"nickname": "昵称",
-	"profile_page": {
-		"role": {
-			"ADMIN": "管理员",
-			"USER": "普通用户",
-			"OWNER": "所有者"
-		}
-	},
 	"backend": {
 		"error": {
 			"incorrect_password": "密码错误。",

packages/next/lib/db/user.ts

@@ -1,16 +1,15 @@
 import { UserType, sqlCred } from "@cvsa/core";
-import { UserProfile } from "../userAuth";
 
 export const getUserBySession = async (sessionID: string) => {
 	const users = await sqlCred<UserType[]>`
-        SELECT user_id as id, username, nickname, "role", user_created_at as created_at
+	    SELECT u.*
-        FROM get_user_by_session_func(${sessionID});
+        FROM users u
+        JOIN login_sessions ls ON u.id = ls.uid
+        WHERE ls.id = ${sessionID};
     `;
-
 	if (users.length === 0) {
 		return undefined;
 	}
-
 	const user = users[0];
 	return {
 		uid: user.id,
@@ -21,36 +20,4 @@ export const getUserBySession = async (sessionID: string) => {
 	};
 };
 
-export const queryUserProfile = async (uid: number, sessionID?: string): Promise<UserProfile | null> => {
+export const getUserSessions = async (sessionID: string) => {};
-	interface Result extends UserType {
-		logged_in: boolean;
-	}
-	const users = await sqlCred<Result[]>`
-        SELECT
-            u.id, u.username, u.nickname, u."role", u.created_at,
-            CASE
-                WHEN (ls.uid IS NOT NULL AND ls.deactivated_at IS NULL AND ls.expire_at > NOW()) THEN TRUE
-                ELSE FALSE
-            END AS logged_in
-        FROM
-            users u
-        LEFT JOIN
-            login_sessions ls ON ls.uid = u.id AND ls.id = ${sessionID || ""}
-        WHERE
-            u.id = ${uid};
-    `;
-
-	if (users.length === 0) {
-		return null;
-	}
-
-	const user = users[0];
-	return {
-		uid: user.id,
-		username: user.username,
-		nickname: user.nickname,
-		role: user.role,
-		createdAt: user.created_at,
-		isLoggedIn: user.logged_in
-	};
-};

packages/next/lib/net.ts

@@ -1,4 +1,4 @@
-import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios";
+import axios, { AxiosRequestConfig, AxiosError, Method } from "axios";
 
 export class ApiRequestError extends Error {
 	public code: number | undefined;
@@ -21,20 +21,10 @@ const httpMethods = {
 	patch: axios.patch
 };
 
-export function fetcher(url: string): Promise<unknown>;
-export function fetcher<JSON = unknown>(
-	url: string,
-	init?: Omit<AxiosRequestConfig, "method"> & { method?: Exclude<HttpMethod, "DELETE"> }
-): Promise<JSON>;
-export function fetcher(
-	url: string,
-	init: Omit<AxiosRequestConfig, "method"> & { method: "DELETE" }
-): Promise<AxiosResponse>;
-
 export async function fetcher<JSON = unknown>(
 	url: string,
 	init?: Omit<AxiosRequestConfig, "method"> & { method?: HttpMethod }
-): Promise<JSON | AxiosResponse<any, any>> {
+): Promise<JSON> {
 	const { method = "get", data, ...config } = init || {};
 
 	const fullConfig: AxiosRequestConfig = {
@@ -48,9 +38,6 @@ export async function fetcher<JSON = unknown>(
 		if (["post", "patch", "put"].includes(m)) {
 			const response = await httpMethods[m](url, data, fullConfig);
 			return response.data;
-		} else if (m === "delete") {
-			const response = await axios.delete(url, fullConfig);
-			return response;
 		} else {
 			const response = await httpMethods[m](url, fullConfig);
 			return response.data;

packages/next/lib/userAuth.ts

@@ -1,16 +1,12 @@
 import { cookies } from "next/headers";
-import { getUserBySession, queryUserProfile } from "@/lib/db/user";
+import { getUserBySession } from "@/lib/db/user";
 
 export interface User {
 	uid: number;
 	username: string;
 	nickname: string | null;
 	role: string;
-	createdAt: Date;
+	createdAt: string;
-}
-
-export interface UserProfile extends User {
-	isLoggedIn: boolean;
 }
 
 export async function getCurrentUser(): Promise<User | null> {
@@ -23,21 +19,6 @@ export async function getCurrentUser(): Promise<User | null> {
 
 		return user ?? null;
 	} catch (error) {
-		console.log(error);
-		return null;
-	}
-}
-
-export async function getUserProfile(uid: number): Promise<UserProfile | null> {
-	const cookieStore = await cookies();
-	const sessionID = cookieStore.get("session_id");
-
-	try {
-		const user = await queryUserProfile(uid, sessionID?.value);
-
-		return user ?? null;
-	} catch (error) {
-		console.log(error);
 		return null;
 	}
 }

packages/next/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "next",
-	"version": "2.9.1",
+	"version": "0.1.0",
 	"private": true,
 	"scripts": {
 		"dev": "next dev --turbopack -p 7400",
@@ -11,19 +11,16 @@
 	},
 	"dependencies": {
 		"@cvsa/backend": "^0.5.3",
-		"@cvsa/core": "0.0.10",
+		"@cvsa/core": "^0.0.7",
 		"@mdx-js/loader": "^3.1.0",
 		"@mdx-js/react": "^3.1.0",
 		"@next/mdx": "^15.3.3",
-		"@types/luxon": "^3.6.2",
 		"@types/mdx": "^2.0.13",
 		"axios": "^1.9.0",
 		"date-fns": "^4.1.0",
 		"framer-motion": "^12.15.0",
 		"fumadocs-mdx": "^11.6.6",
 		"i18next": "^25.2.1",
-		"jotai": "^2.12.5",
-		"luxon": "^3.6.1",
 		"next": "^15.3.3",
 		"next-intl": "^4.1.0",
 		"raw-loader": "^4.0.2",
@@ -31,8 +28,7 @@
 		"react-dom": "^19.1.0",
 		"swr": "^2.3.3",
 		"ua-parser-js": "^2.0.3",
-		"yup": "^1.6.1",
+		"yup": "^1.6.1"
-		"yup-numeric": "^0.5.0"
 	},
 	"devDependencies": {
 		"typescript": "^5.8.3",

src/db/raw/fetchAids.ts

@@ -0,0 +1,121 @@
+import { Database } from "jsr:@db/sqlite@0.12";
+import { ensureDir } from "https://deno.land/std@0.113.0/fs/mod.ts";
+
+// 常量定义
+const MAX_RETRIES = 3;
+const API_URL = "https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=50&pn=";
+const DATABASE_PATH = "./data/main.db";
+const LOG_DIR = "./logs/bili-info-crawl";
+const LOG_FILE = `${LOG_DIR}/run-${Date.now() / 1000}.log`;
+
+// 打开数据库
+const db = new Database(DATABASE_PATH, { int64: true });
+
+// 设置日志
+async function setupLogging() {
+	await ensureDir(LOG_DIR);
+	const logStream = await Deno.open(LOG_FILE, {
+		write: true,
+		create: true,
+		append: true,
+	});
+
+	const redirectConsole =
+		// deno-lint-ignore no-explicit-any
+		(originalConsole: (...args: any[]) => void) =>
+		// deno-lint-ignore no-explicit-any
+		(...args: any[]) => {
+			const message = args.map((
+				arg,
+			) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
+			originalConsole(message);
+			logStream.write(new TextEncoder().encode(message + "\n"));
+		};
+
+	console.log = redirectConsole(console.log);
+	console.error = redirectConsole(console.error);
+	console.warn = redirectConsole(console.warn);
+}
+
+interface Metadata {
+	key: string;
+	value: string;
+}
+
+// 获取最后一次更新的时间
+function getLastUpdate(): Date {
+	const result = db.prepare(
+		"SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'",
+	).get() as Metadata;
+	return result ? new Date(result.value as string) : new Date(0);
+}
+
+// 更新最后更新时间
+function updateLastUpdate() {
+	const now = new Date().toISOString();
+	db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'")
+		.run(now);
+}
+
+// 辅助函数：获取数据
+// deno-lint-ignore no-explicit-any
+async function fetchData(pn: number, retries = MAX_RETRIES): Promise<any> {
+	try {
+		const response = await fetch(`${API_URL}${pn}`);
+		if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
+		return await response.json();
+	} catch (error) {
+		if (retries > 0) {
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+			return fetchData(pn, retries - 1);
+		}
+		throw error;
+	}
+}
+
+// 插入 aid 到数据库
+function insertAid(aid: number) {
+	db.prepare(
+		"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
+	).run(aid);
+}
+
+// 主函数
+async function main() {
+	await setupLogging();
+
+	let pn = 1;
+	let shouldContinue = true;
+	const lastUpdate = getLastUpdate();
+
+	while (shouldContinue) {
+		try {
+			const data = await fetchData(pn);
+			const archives = data.data.archives;
+
+			for (const archive of archives) {
+				const pubTime = new Date(archive.pubdate * 1000);
+				if (pubTime > lastUpdate) {
+					insertAid(archive.aid);
+				} else {
+					shouldContinue = false;
+					break;
+				}
+			}
+
+			pn++;
+			console.log(`Fetched page ${pn}`);
+		} catch (error) {
+			console.error(`Error fetching data for pn=${pn}: ${error}`);
+		}
+	}
+
+	// 更新最后更新时间
+	updateLastUpdate();
+
+	// 关闭数据库
+	db.close();
+}
+
+// 运行主函数
+main().catch(console.error);

src/db/raw/insertAidsToDB.ts

@@ -0,0 +1,223 @@
+import path from "node:path";
+import { Database } from "jsr:@db/sqlite@0.12";
+import { getBiliBiliVideoInfo } from "./videoInfo.ts";
+import { ensureDir } from "https://deno.land/std@0.113.0/fs/mod.ts";
+
+const aidPath = "./data/2025010104_c30_aids.txt";
+const db = new Database("./data/main.db", { int64: true });
+const regions = [
+	"shanghai",
+	"hangzhou",
+	"qingdao",
+	"beijing",
+	"zhangjiakou",
+	"chengdu",
+	"shenzhen",
+	"hohhot",
+];
+const logDir = "./logs/bili-info-crawl";
+const logFile = path.join(logDir, `run-${Date.now() / 1000}.log`);
+const shouldReadTextFile = false;
+
+const SECOND = 1000;
+const SECONDS = SECOND;
+const MINUTE = 60 * SECONDS;
+const MINUTES = MINUTE;
+const IPs = regions.length;
+
+const rateLimits = [
+	{ window: 5 * MINUTES, maxRequests: 160 * IPs },
+	{ window: 30 * SECONDS, maxRequests: 20 * IPs },
+	{ window: 1.2 * SECOND, maxRequests: 1 * IPs },
+];
+
+const requestQueue: number[] = [];
+
+async function setupLogging() {
+	await ensureDir(logDir);
+	const logStream = await Deno.open(logFile, {
+		write: true,
+		create: true,
+		append: true,
+	});
+
+	const redirectConsole =
+		// deno-lint-ignore no-explicit-any
+		(originalConsole: (...args: any[]) => void) =>
+		// deno-lint-ignore no-explicit-any
+		(...args: any[]) => {
+			const message = args.map((
+				arg,
+			) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
+			originalConsole(message);
+			logStream.write(new TextEncoder().encode(message + "\n"));
+		};
+
+	console.log = redirectConsole(console.log);
+	console.error = redirectConsole(console.error);
+	console.warn = redirectConsole(console.warn);
+}
+
+function isRateLimited(): boolean {
+	const now = Date.now();
+	return rateLimits.some(({ window, maxRequests }) => {
+		const windowStart = now - window;
+		const requestsInWindow = requestQueue.filter((timestamp) => timestamp >= windowStart).length;
+		return requestsInWindow >= maxRequests;
+	});
+}
+
+async function readFromText() {
+	const aidRawcontent = await Deno.readTextFile(aidPath);
+	const aids = aidRawcontent
+		.split("\n")
+		.filter((line) => line.length > 0)
+		.map((line) => parseInt(line));
+
+	// if (!db.prepare("SELECT COUNT(*) FROM bili_info_crawl").get()) {
+	//     const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')");
+	//     aids.forEach((aid) => insertStmt.run(aid));
+	// }
+
+	// 查询数据库中已经存在的 aid
+	const existingAids = db
+		.prepare("SELECT aid FROM bili_info_crawl")
+		.all()
+		.map((row) => row.aid);
+	console.log(existingAids.length);
+
+	// 将 existingAids 转换为 Set 以提高查找效率
+	const existingAidsSet = new Set(existingAids);
+
+	// 找出 aids 数组中不存在于数据库的条目
+	const newAids = aids.filter((aid) => !existingAidsSet.has(aid));
+
+	// 插入这些新条目
+	const insertStmt = db.prepare(
+		"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
+	);
+	newAids.forEach((aid) => insertStmt.run(aid));
+}
+
+async function insertAidsToDB() {
+	if (shouldReadTextFile) {
+		await readFromText();
+	}
+
+	const aidsInDB = db
+		.prepare(
+			"SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'",
+		)
+		.all()
+		.map((row) => row.aid) as number[];
+
+	const totalAids = aidsInDB.length;
+	let processedAids = 0;
+	const startTime = Date.now();
+
+	const processAid = async (aid: number) => {
+		try {
+			const res = await getBiliBiliVideoInfo(
+				aid,
+				regions[processedAids % regions.length],
+			);
+			if (res === null) {
+				updateAidStatus(aid, "failed");
+			} else {
+				const rawData = JSON.parse(res);
+				if (rawData.code === 0) {
+					updateAidStatus(
+						aid,
+						"success",
+						rawData.data.View.bvid,
+						JSON.stringify(rawData.data),
+					);
+				} else {
+					updateAidStatus(aid, "error", undefined, res);
+				}
+			}
+		} catch (error) {
+			console.error(`Error updating aid ${aid}: ${error}`);
+			updateAidStatus(aid, "failed");
+		} finally {
+			processedAids++;
+			logProgress(aid, processedAids, totalAids, startTime);
+		}
+	};
+
+	const interval = setInterval(async () => {
+		if (aidsInDB.length === 0) {
+			clearInterval(interval);
+			console.log("All aids processed.");
+			return;
+		}
+		if (!isRateLimited()) {
+			const aid = aidsInDB.shift();
+			if (aid !== undefined) {
+				requestQueue.push(Date.now());
+				await processAid(aid);
+			}
+		}
+	}, 50);
+
+	console.log("Starting to process aids...");
+}
+
+function updateAidStatus(
+	aid: number,
+	status: string,
+	bvid?: string,
+	data?: string,
+) {
+	const stmt = db.prepare(`
+        UPDATE bili_info_crawl
+        SET status = ?,
+        ${bvid ? "bvid = ?," : ""}
+        ${data ? "data = ?," : ""}
+        timestamp = ?
+        WHERE aid = ?
+    `);
+	const params = [
+		status,
+		...(bvid ? [bvid] : []),
+		...(data ? [data] : []),
+		Date.now() / 1000,
+		aid,
+	];
+	stmt.run(...params);
+}
+
+function logProgress(
+	aid: number,
+	processedAids: number,
+	totalAids: number,
+	startTime: number,
+) {
+	const elapsedTime = Date.now() - startTime;
+	const elapsedSeconds = Math.floor(elapsedTime / 1000);
+	const elapsedMinutes = Math.floor(elapsedSeconds / 60);
+	const elapsedHours = Math.floor(elapsedMinutes / 60);
+
+	const remainingAids = totalAids - processedAids;
+	const averageTimePerAid = elapsedTime / processedAids;
+	const eta = remainingAids * averageTimePerAid;
+	const etaSeconds = Math.floor(eta / 1000);
+	const etaMinutes = Math.floor(etaSeconds / 60);
+	const etaHours = Math.floor(etaMinutes / 60);
+
+	const progress = `${processedAids}/${totalAids}, ${
+		((processedAids / totalAids) * 100).toFixed(
+			2,
+		)
+	}%, elapsed ${elapsedHours.toString().padStart(2, "0")}:${(elapsedMinutes % 60).toString().padStart(2, "0")}:${
+		(
+			elapsedSeconds % 60
+		)
+			.toString()
+			.padStart(2, "0")
+	}, ETA ${etaHours}h${(etaMinutes % 60).toString().padStart(2, "0")}m`;
+	console.log(`Updated aid ${aid}, ${progress}`);
+}
+
+await setupLogging();
+insertAidsToDB();

src/db/raw/videoInfo.ts

@@ -0,0 +1,60 @@
+export async function getBiliBiliVideoInfo(
+	bvidORaid?: string | number,
+	region: string = "hangzhou",
+) {
+	const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined;
+	const aid = typeof bvidORaid === "number" ? bvidORaid : undefined;
+
+	const baseURL = "https://api.bilibili.com/x/web-interface/view/detail";
+	const urlObject = new URL(baseURL);
+
+	if (aid) {
+		urlObject.searchParams.append("aid", aid.toString());
+		const finalURL = urlObject.toString();
+		return await proxyRequestWithRegion(finalURL, region);
+	} else if (bvid) {
+		urlObject.searchParams.append("bvid", bvid);
+		const finalURL = urlObject.toString();
+		return await proxyRequestWithRegion(finalURL, region);
+	} else {
+		return null;
+	}
+}
+
+async function proxyRequestWithRegion(
+	url: string,
+	region: string,
+): Promise<any | null> {
+	const td = new TextDecoder();
+	// aliyun configure set  --access-key-id $ALIYUN_AK --access-key-secret $ALIYUN_SK --region cn-shenzhen --profile CVSA-shenzhen --mode AK
+	const p = await new Deno.Command("aliyun", {
+		args: [
+			"fc",
+			"POST",
+			`/2023-03-30/functions/proxy-${region}/invocations`,
+			"--qualifier",
+			"LATEST",
+			"--header",
+			"Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;",
+			"--body",
+			JSON.stringify({ url: url }),
+			"--profile",
+			`CVSA-${region}`,
+		],
+	}).output();
+	try {
+		const out = td.decode(p.stdout);
+		const rawData = JSON.parse(out);
+		if (rawData.statusCode !== 200) {
+			console.error(
+				`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`,
+			);
+			return null;
+		} else {
+			return JSON.parse(rawData.body);
+		}
+	} catch (e) {
+		console.error(`Error proxying request ${url} to ${region}: ${e}`);
+		return null;
+	}
+}

src/metadataArchive.ts

@@ -1,32 +0,0 @@
-import arg from "arg";
-//import { getVideoDetails } from "@crawler/net/getVideoDetails";
-import logger from "@core/log/logger";
-
-const quit = (reason: string) => {
-	logger.error(reason);
-	process.exit();
-};
-
-const args = arg({
-	"--aids": String // --port <number> or --port=<number>
-});
-
-const aidsFileName = args["--aids"];
-
-if (!aidsFileName) {
-	quit("Missing --aids <file_path>");
-}
-
-const aidsFile = Bun.file(aidsFileName!);
-const fileExists = await aidsFile.exists();
-if (!fileExists) {
-	quit(`${aidsFile} does not exist.`);
-}
-
-const aidsText = await aidsFile.text();
-const aids = aidsText
-	.split("\n")
-	.map((line) => parseInt(line))
-	.filter((num) => !Number.isNaN(num));
-
-logger.log(`Read ${aids.length} aids.`);

tsconfig.json

@@ -1,35 +0,0 @@
-{
-	"include": ["**/*.ts"],
-	"compilerOptions": {
-		"baseUrl": ".",
-		"paths": {
-			"@core/*": ["./packages/core/*"],
-			"@crawler/*": ["./packages/crawler/*"]
-		},
-		// Environment setup & latest features
-		"lib": ["ESNext"],
-		"target": "ESNext",
-		"module": "Preserve",
-		"moduleDetection": "force",
-		"jsx": "react-jsx",
-		"allowJs": true,
-
-		// Bundler mode
-		"moduleResolution": "bundler",
-		"allowImportingTsExtensions": true,
-		"verbatimModuleSyntax": true,
-		"noEmit": true,
-
-		// Best practices
-		"strict": true,
-		"skipLibCheck": true,
-		"noFallthroughCasesInSwitch": true,
-		"noUncheckedIndexedAccess": true,
-		"noImplicitOverride": true,
-
-		// Some stricter flags (disabled by default)
-		"noUnusedLocals": false,
-		"noUnusedParameters": false,
-		"noPropertyAccessFromIndexSignature": false
-	}
-}