diff --git a/packages/backend/deno.json b/packages/backend/deno.json index 76b3f5a..eeabe3a 100644 --- a/packages/backend/deno.json +++ b/packages/backend/deno.json @@ -5,11 +5,13 @@ "hono": "jsr:@hono/hono@^4.7.5", "zod": "npm:zod", "yup": "npm:yup", - "@core/": "../core/" + "@core/": "../core/", + "@crawler/net/videoInfo": "../crawler/net/getVideoInfo.ts", + "ioredis": "npm:ioredis" }, "tasks": { - "dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts", - "start": "deno serve --env-file=.env --allow-env --allow-net --host 127.0.0.1 main.ts" + "dev": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --watch main.ts", + "start": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --host 127.0.0.1 main.ts" }, "compilerOptions": { "jsx": "precompile", diff --git a/packages/backend/main.ts b/packages/backend/main.ts index 6746129..5cbf42f 100644 --- a/packages/backend/main.ts +++ b/packages/backend/main.ts @@ -3,6 +3,7 @@ import { dbCredMiddleware, dbMiddleware } from "./database.ts"; import { rootHandler } from "./root.ts"; import { getSnapshotsHanlder } from "./snapshots.ts"; import { registerHandler } from "./register.ts"; +import { videoInfoHandler } from "./videoInfo.ts"; export const app = new Hono(); @@ -14,10 +15,12 @@ app.get("/", ...rootHandler); app.get("/video/:id/snapshots", ...getSnapshotsHanlder); app.post("/user", ...registerHandler); +app.get("/video/:id/info", ...videoInfoHandler); + const fetch = app.fetch; export default { fetch, } satisfies Deno.ServeDefaultExport; -export const VERSION = "0.3.0"; +export const VERSION = "0.4.2"; diff --git a/packages/backend/snapshots.ts b/packages/backend/snapshots.ts index 56a7582..7d191d5 100644 --- a/packages/backend/snapshots.ts +++ b/packages/backend/snapshots.ts @@ -12,7 +12,7 @@ const SnapshotQueryParamsSchema = object({ reverse: boolean().optional(), }); -const idSchema = mixed().test( +export const idSchema = mixed().test( "is-valid-id", 'id must be a string starting with "av" followed by digits, or "BV" followed by 10 alphanumeric characters, or a positive integer', async (value) => { diff --git a/packages/backend/videoInfo.ts b/packages/backend/videoInfo.ts new file mode 100644 index 0000000..836c8e6 --- /dev/null +++ b/packages/backend/videoInfo.ts @@ -0,0 +1,85 @@ +import type { Context } from "hono"; +import { createHandlers } from "./utils.ts"; +import type { BlankEnv, BlankInput } from "hono/types"; +import { number, ValidationError } from "yup"; +import { getVideoInfo, getVideoInfoByBV } from "@crawler/net/videoInfo"; +import { idSchema } from "./snapshots.ts"; +import type { VideoInfoData } from "../crawler/net/bilibili.d.ts"; +import { Redis } from "ioredis"; +import { NetSchedulerError } from "../crawler/mq/scheduler.ts"; +import logger from "../crawler/log/logger.ts"; +import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; + +const redis = new Redis({ maxRetriesPerRequest: null }); +const CACHE_EXPIRATION_SECONDS = 60; + +type ContextType = Context; + +async function insertVideoSnapshot(client: Client, data: VideoInfoData) { + const views = data.stat.view; + const danmakus = data.stat.danmaku; + const replies = data.stat.reply; + const likes = data.stat.like; + const coins = data.stat.coin; + const shares = data.stat.share; + const favorites = data.stat.favorite; + const aid = data.aid; + + const query: string = ` + INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + `; + + await client.queryObject( + query, + [aid, views, danmakus, replies, likes, coins, shares, favorites], + ); + + logger.log(`Inserted into snapshot for video ${aid} by videoInfo API.`, "api", "fn:insertVideoSnapshot"); +} + + +export const videoInfoHandler = createHandlers(async (c: ContextType) => { + const client = c.get("db"); + try { + const id = await idSchema.validate(c.req.param("id")); + let videoId: string | number = id as string; + if (videoId.startsWith("av")) { + videoId = parseInt(videoId.slice(2)); + } else if (await number().isValid(videoId)) { + videoId = parseInt(videoId); + } + + const cacheKey = `cvsa:videoInfo:${videoId}`; + const cachedData = await redis.get(cacheKey); + + if (cachedData) { + return c.json(JSON.parse(cachedData)); + } + + let result: VideoInfoData | number; + if (typeof videoId === "number") { + result = await getVideoInfo(videoId, "getVideoInfo"); + } else { + result = await getVideoInfoByBV(videoId, "getVideoInfo"); + } + + if (typeof result === "number") { + return c.json({ message: "Error fetching video info", code: result }, 500); + } + + await redis.setex(cacheKey, CACHE_EXPIRATION_SECONDS, JSON.stringify(result)); + + await insertVideoSnapshot(client, result); + + return c.json(result); + } catch (e) { + if (e instanceof ValidationError) { + return c.json({ message: "Invalid query parameters", errors: e.errors }, 400); + } else if (e instanceof NetSchedulerError) { + return c.json({ message: "Error fetching video info", code: e.code }, 500); + } else { + return c.json({ message: "Unhandled error", error: e }, 500); + } + } +}); \ No newline at end of file diff --git a/packages/crawler/net/getVideoInfo.ts b/packages/crawler/net/getVideoInfo.ts index b12f5c0..801e2ad 100644 --- a/packages/crawler/net/getVideoInfo.ts +++ b/packages/crawler/net/getVideoInfo.ts @@ -25,3 +25,27 @@ export async function getVideoInfo(aid: number, task: string): Promise} VideoInfoData or the error code returned by bilibili API + * @throws {NetSchedulerError} - The error will be thrown in following cases: + * - No proxy is available currently: with error code `NO_PROXY_AVAILABLE` + * - The native `fetch` function threw an error: with error code `FETCH_ERROR` + * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR` + */ +export async function getVideoInfoByBV(bvid: string, task: string): Promise { + const url = `https://api.bilibili.com/x/web-interface/view?bvid=${bvid}`; + const data = await netScheduler.request(url, task); + const errMessage = `Error fetching metadata for ${bvid}:`; + if (data.code !== 0) { + logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfoByBV"); + return data.code; + } + return data.data; +} \ No newline at end of file