diff --git a/lib/db/snapshotSchedule.ts b/lib/db/snapshotSchedule.ts index 8c825b7..158fef9 100644 --- a/lib/db/snapshotSchedule.ts +++ b/lib/db/snapshotSchedule.ts @@ -271,6 +271,13 @@ export async function setSnapshotStatus(client: Client, id: number, status: stri ); } +export async function bulkSetSnapshotStatus(client: Client, ids: number[], status: string) { + return await client.queryObject( + `UPDATE snapshot_schedule SET status = $2 WHERE id = ANY($1)`, + [ids, status], + ); +} + export async function getVideosWithoutActiveSnapshotSchedule(client: Client) { const query: string = ` SELECT s.aid diff --git a/lib/mq/exec/snapshotTick.ts b/lib/mq/exec/snapshotTick.ts index e2c4ccf..f29dfce 100644 --- a/lib/mq/exec/snapshotTick.ts +++ b/lib/mq/exec/snapshotTick.ts @@ -2,6 +2,7 @@ import { Job } from "bullmq"; import { db } from "lib/db/init.ts"; import { getLatestVideoSnapshot, getVideosNearMilestone } from "lib/db/snapshot.ts"; import { + bulkSetSnapshotStatus, findClosestSnapshot, findSnapshotBefore, getLatestSnapshot, @@ -23,6 +24,7 @@ import { getBiliVideoStatus, setBiliVideoStatus } from "lib/db/allData.ts"; import { truncate } from "lib/utils/truncate.ts"; import { lockManager } from "lib/mq/lockManager.ts"; import { getSongsPublihsedAt } from "lib/db/songs.ts"; +import { bulkGetVideoStats } from "lib/net/bulkGetVideoStats.ts"; const priorityMap: { [key: string]: number } = { "milestone": 1, @@ -187,6 +189,31 @@ export const regularSnapshotsWorker = async (_job: Job) => { } }; +export const takeBulkSnapshotForVideosWorker = async (job: Job) => { + const dataMap: {[key: number]: number} = job.data.map; + const ids = Object.keys(dataMap).map((id) => Number(id)); + const aidsToFetch: number[] = []; + const client = await db.connect(); + try { + for (const id of ids) { + const aid = Number(dataMap[id]); + const exists = await snapshotScheduleExists(client, id); + if (!exists) { + continue + } + aidsToFetch.push(aid); + } + const data = await bulkGetVideoStats(aidsToFetch); + if (typeof data === "number") { + await bulkSetSnapshotStatus(client, ids, "failed"); + return `GET_BILI_STATUS_${data}`; + } + } + finally { + client.release(); + } +} + export const takeSnapshotForVideoWorker = async (job: Job) => { const id = job.data.id; const aid = Number(job.data.aid); diff --git a/lib/mq/scheduler.ts b/lib/mq/scheduler.ts index c31d8ef..6345755 100644 --- a/lib/mq/scheduler.ts +++ b/lib/mq/scheduler.ts @@ -339,6 +339,12 @@ bili_test[1].max = 36; bili_test[2].max = 150; bili_test[3].max = 1000; +const bili_strict = biliLimiterConfig; +bili_strict[0].max = 4; +bili_strict[1].max = 8; +bili_strict[2].max = 30; +bili_strict[3].max = 100; + /* Execution order for setup: @@ -378,11 +384,21 @@ netScheduler.addTask("snapshotVideo", "bili_test", [ "alicloud-shenzhen", "alicloud-hohhot", ]); +netScheduler.addTask("bulkSnapshot", "bili_strict", [ + "alicloud-qingdao", + "alicloud-shanghai", + "alicloud-zhangjiakou", + "alicloud-chengdu", + "alicloud-shenzhen", + "alicloud-hohhot", +]); netScheduler.setTaskLimiter("getVideoInfo", videoInfoRateLimiterConfig); netScheduler.setTaskLimiter("getLatestVideos", null); netScheduler.setTaskLimiter("snapshotMilestoneVideo", null); netScheduler.setTaskLimiter("snapshotVideo", null); +netScheduler.setTaskLimiter("bulkSnapshot", null); netScheduler.setProviderLimiter("bilibili", biliLimiterConfig); netScheduler.setProviderLimiter("bili_test", bili_test); +netScheduler.setProviderLimiter("bili_strict", bili_strict); export default netScheduler; diff --git a/lib/net/bilibili.d.ts b/lib/net/bilibili.d.ts index 964a87e..19e1ba2 100644 --- a/lib/net/bilibili.d.ts +++ b/lib/net/bilibili.d.ts @@ -11,10 +11,11 @@ export type VideoTagsResponse = BaseResponse; export type VideoInfoResponse = BaseResponse; export type MediaListInfoResponse = BaseResponse; -type MediaListInfoData = MediaListInfoItem[]; +export type MediaListInfoData = MediaListInfoItem[]; -interface MediaListInfoItem { +export interface MediaListInfoItem { + attr: number; bvid: string; id: number; cnt_info: { diff --git a/lib/net/bulkGetVideoStats.ts b/lib/net/bulkGetVideoStats.ts new file mode 100644 index 0000000..7240bed --- /dev/null +++ b/lib/net/bulkGetVideoStats.ts @@ -0,0 +1,27 @@ +import netScheduler from "lib/mq/scheduler.ts"; +import { MediaListInfoData, MediaListInfoResponse } from "lib/net/bilibili.d.ts"; +import logger from "lib/log/logger.ts"; + +/* + * Bulk fetch video metadata from bilibili API + * @param {number[]} aids - The aid list to fetch + * @returns {Promise} MediaListInfoData or the error code returned by bilibili API + * @throws {NetSchedulerError} - The error will be thrown in following cases: + * - No proxy is available currently: with error code `NO_PROXY_AVAILABLE` + * - The native `fetch` function threw an error: with error code `FETCH_ERROR` + * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR` + */ +export async function bulkGetVideoStats(aids: number[]): Promise { + const baseURL = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`; + let url = baseURL; + for (const aid of aids) { + url += `${aid}:2,`; + } + const data = await netScheduler.request(url, "bulkSnapshot"); + const errMessage = `Error fetching metadata for aid list: ${aids.join(",")}:`; + if (data.code !== 0) { + logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfo"); + return data.code; + } + return data.data; +}