merge: branch 'feat/backend' into ref/structure

packages/backend/deno.json

@@ -5,11 +5,13 @@
 		"hono": "jsr:@hono/hono@^4.7.5",
 		"zod": "npm:zod",
 		"yup": "npm:yup",
-		"@core/": "../core/"
+		"@core/": "../core/",
+		"@crawler/net/videoInfo": "../crawler/net/getVideoInfo.ts",
+		"ioredis": "npm:ioredis"
 	},
 	"tasks": {
-		"dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts",
+		"dev": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --watch main.ts",
-		"start": "deno serve --env-file=.env --allow-env --allow-net --host 127.0.0.1 main.ts"
+		"start": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --host 127.0.0.1 main.ts"
 	},
 	"compilerOptions": {
 		"jsx": "precompile",

packages/backend/main.ts

@@ -3,6 +3,7 @@ import { dbCredMiddleware, dbMiddleware } from "./database.ts";
 import { rootHandler } from "./root.ts";
 import { getSnapshotsHanlder } from "./snapshots.ts";
 import { registerHandler } from "./register.ts";
+import { videoInfoHandler } from "./videoInfo.ts";
 
 export const app = new Hono();
 
@@ -14,10 +15,12 @@ app.get("/", ...rootHandler);
 app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
 app.post("/user", ...registerHandler);
 
+app.get("/video/:id/info", ...videoInfoHandler);
+
 const fetch = app.fetch;
 
 export default {
 	fetch,
 } satisfies Deno.ServeDefaultExport;
 
-export const VERSION = "0.3.0";
+export const VERSION = "0.4.2";

packages/backend/snapshots.ts

@@ -12,7 +12,7 @@ const SnapshotQueryParamsSchema = object({
 	reverse: boolean().optional(),
 });
 
-const idSchema = mixed().test(
+export const idSchema = mixed().test(
 	"is-valid-id",
 	'id must be a string starting with "av" followed by digits, or "BV" followed by 10 alphanumeric characters, or a positive integer',
 	async (value) => {

packages/backend/videoInfo.ts

@@ -0,0 +1,85 @@
+import type { Context } from "hono";
+import { createHandlers } from "./utils.ts";
+import type { BlankEnv, BlankInput } from "hono/types";
+import { number, ValidationError } from "yup";
+import { getVideoInfo, getVideoInfoByBV } from "@crawler/net/videoInfo";
+import { idSchema } from "./snapshots.ts";
+import type { VideoInfoData } from "../crawler/net/bilibili.d.ts";
+import { Redis } from "ioredis";
+import { NetSchedulerError } from "../crawler/mq/scheduler.ts";
+import logger from "../crawler/log/logger.ts";
+import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
+
+const redis = new Redis({ maxRetriesPerRequest: null });
+const CACHE_EXPIRATION_SECONDS = 60;
+
+type ContextType = Context<BlankEnv, "/video/:id/info", BlankInput>;
+
+async function insertVideoSnapshot(client: Client, data: VideoInfoData) {
+	const views = data.stat.view;
+	const danmakus = data.stat.danmaku;
+	const replies = data.stat.reply;
+	const likes = data.stat.like;
+	const coins = data.stat.coin;
+	const shares = data.stat.share;
+	const favorites = data.stat.favorite;
+	const aid = data.aid;
+
+	const query: string = `
+        INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
+        VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
+    `;
+
+	await client.queryObject(
+		query,
+		[aid, views, danmakus, replies, likes, coins, shares, favorites],
+	);
+
+	logger.log(`Inserted into snapshot for video ${aid} by videoInfo API.`, "api", "fn:insertVideoSnapshot");
+}
+
+
+export const videoInfoHandler = createHandlers(async (c: ContextType) => {
+	const client = c.get("db");
+	try {
+		const id = await idSchema.validate(c.req.param("id"));
+		let videoId: string | number = id as string;
+		if (videoId.startsWith("av")) {
+			videoId = parseInt(videoId.slice(2));
+		} else if (await number().isValid(videoId)) {
+			videoId = parseInt(videoId);
+		}
+
+		const cacheKey = `cvsa:videoInfo:${videoId}`;
+		const cachedData = await redis.get(cacheKey);
+
+		if (cachedData) {
+			return c.json(JSON.parse(cachedData));
+		}
+
+		let result: VideoInfoData | number;
+		if (typeof videoId === "number") {
+			result = await getVideoInfo(videoId, "getVideoInfo");
+		} else {
+			result = await getVideoInfoByBV(videoId, "getVideoInfo");
+		}
+
+		if (typeof result === "number") {
+			return c.json({ message: "Error fetching video info", code: result }, 500);
+		}
+
+		await redis.setex(cacheKey, CACHE_EXPIRATION_SECONDS, JSON.stringify(result));
+
+		await insertVideoSnapshot(client, result);
+
+		return c.json(result);
+	} catch (e) {
+		if (e instanceof ValidationError) {
+			return c.json({ message: "Invalid query parameters", errors: e.errors }, 400);
+		} else if (e instanceof NetSchedulerError) {
+			return c.json({ message: "Error fetching video info", code: e.code }, 500);
+		} else {
+			return c.json({ message: "Unhandled error", error: e }, 500);
+		}
+	}
+});

packages/crawler/net/getVideoInfo.ts

@@ -25,3 +25,27 @@ export async function getVideoInfo(aid: number, task: string): Promise<VideoInfo
 	}
 	return data.data;
 }
+
+/*
+ * Fetch video metadata from bilibili API by BVID
+ * @param {string} bvid - The video's BVID
+ * @param {string} task - The task name used in scheduler. It can be one of the following:
+ * - snapshotVideo
+ * - getVideoInfo
+ * - snapshotMilestoneVideo
+ * @returns {Promise<VideoInfoData | number>} VideoInfoData or the error code returned by bilibili API
+ * @throws {NetSchedulerError} - The error will be thrown in following cases:
+ * - No proxy is available currently: with error code `NO_PROXY_AVAILABLE`
+ * - The native `fetch` function threw an error: with error code `FETCH_ERROR`
+ * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
+ */
+export async function getVideoInfoByBV(bvid: string, task: string): Promise<VideoInfoData | number> {
+	const url = `https://api.bilibili.com/x/web-interface/view?bvid=${bvid}`;
+	const data = await netScheduler.request<VideoInfoResponse>(url, task);
+	const errMessage = `Error fetching metadata for ${bvid}:`;
+	if (data.code !== 0) {
+		logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfoByBV");
+		return data.code;
+	}
+	return data.data;
+}