diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml index a230540..ef8d276 100644 --- a/.idea/data_source_mapping.xml +++ b/.idea/data_source_mapping.xml @@ -1,7 +1,8 @@ - - + + + \ No newline at end of file diff --git a/.idea/db-forest-config.xml b/.idea/db-forest-config.xml index d092723..2d44b8c 100644 --- a/.idea/db-forest-config.xml +++ b/.idea/db-forest-config.xml @@ -1,6 +1,6 @@ - \ No newline at end of file diff --git a/packages/core/net/getVideoDetails.ts b/packages/core/net/getVideoDetails.ts index 5de4be5..aaefd12 100644 --- a/packages/core/net/getVideoDetails.ts +++ b/packages/core/net/getVideoDetails.ts @@ -1,6 +1,6 @@ import logger from "@core/log"; import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d"; -import networkDelegate from "@core/net/delegate"; +import networkDelegate, { type RequestTasks } from "@core/net/delegate"; /** * Fetch detailed video metadata from bilibili API @@ -8,9 +8,12 @@ import networkDelegate from "@core/net/delegate"; * @returns The detailed metadata of the video, or null if the video does not exist * @throws {NetSchedulerError} The caller would need to handle this error */ -export async function getVideoDetails(aid: number): Promise { +export async function getVideoDetails( + aid: number, + task: RequestTasks = "getVideoInfo" +): Promise { const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`; - const { data } = await networkDelegate.request(url, "getVideoInfo"); + const { data } = await networkDelegate.request(url, task); const errMessage = `Error fetching metadata for ${aid}:`; if (data.code !== 0) { logger.error(`${errMessage + data.code}-${data.message}`, "net", "fn:getVideoInfo"); diff --git a/src/fullSnapshot.ts b/src/fullSnapshot.ts index 4317680..07301f7 100644 --- a/src/fullSnapshot.ts +++ b/src/fullSnapshot.ts @@ -27,7 +27,7 @@ const getAids = async () => { return (await Bun.file(aidsFile).text()).split("\n").map(Number); } const aids = await sql<{ aid: number }[]>`SELECT aid FROM bilibili_metadata`; - return aids.map((row: any) => row.aid); + return aids.map((row) => row.aid); }; async function addCandidates() { @@ -36,9 +36,9 @@ async function addCandidates() { logger.log(`Retrieved ${aids.length} from production DB.`); const existingAids = sqlite - .prepare("SELECT aid FROM bili_info_crawl") + .prepare<{ aid: number }, []>("SELECT aid FROM bili_info_crawl") .all() - .map((row: any) => row.aid); + .map((row) => row.aid); logger.log(`We have ${existingAids.length} from local DB.`); const existingAidsSet = new Set(existingAids); @@ -46,30 +46,30 @@ async function addCandidates() { const newAids = aids.filter((aid) => !existingAidsSet.has(aid)); const insertStmt = sqlite.prepare( - `INSERT INTO bili_info_crawl (aid, status) VALUES (?, 'pending') ON CONFLICT DO NOTHING;` - ); + `INSERT INTO bili_info_crawl (aid, status) VALUES (?, 'pending') ON CONFLICT DO NOTHING;` + ); const insertMany = sqlite.transaction((data) => { - for (const aid of data) { - insertStmt.run(aid); - } - }); + for (const aid of data) { + insertStmt.run(aid); + } + }); try { - insertMany(newAids); - logger.log(`Successfully added ${newAids.length} to local DB.`); - } catch (err) { - logger.error(["Failed to insert candidates:", err]); - } + insertMany(newAids); + logger.log(`Successfully added ${newAids.length} to local DB.`); + } catch (err) { + logger.error(["Failed to insert candidates:", err]); + } } async function insertAidsToDB() { await addCandidates(); const aidsInDB = sqlite - .prepare("SELECT aid FROM bili_info_crawl WHERE status = 'pending'") + .prepare<{ aid: number }, []>("SELECT aid FROM bili_info_crawl WHERE status = 'pending'") .all() - .map((row: any) => row.aid) as number[]; + .map((row) => row.aid) as number[]; const totalAids = aidsInDB.length; let processedAids = 0; @@ -77,7 +77,7 @@ async function insertAidsToDB() { const processAid = async (aid: number) => { try { - const res = await getVideoDetails(aid); + const res = await getVideoDetails(aid, "annualArchive"); if (res === null) { updateAidStatus(aid, "failed"); } else {