alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: don't adjust start time for schedule when no proxy is available

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-06-06 17:11:44 +08:00
parent 1e8d28e194
commit 8cf9395354
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
3 changed files with 22 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
packages/crawler/db/snapshotSchedule.ts
View File

@ -182,7 +182,8 @@ export async function scheduleSnapshot(
aid: number, aid: number,
type: string, type: string,
targetTime: number, targetTime: number,
force: boolean = false force: boolean = false,
adjustTime: boolean = true
) { ) {
let adjustedTime = new Date(targetTime); let adjustedTime = new Date(targetTime);
const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type); const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type);
@ -204,7 +205,7 @@ export async function scheduleSnapshot(
} }
} }
if (hashActiveSchedule && !force) return; if (hashActiveSchedule && !force) return;
if (type !== "milestone" && type !== "new") { if (type !== "milestone" && type !== "new" && adjustTime) {
adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis); adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis);
} }
logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot"); logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot");
@ -224,10 +225,11 @@ export async function bulkScheduleSnapshot(
aids: number[], aids: number[],
type: string, type: string,
targetTime: number, targetTime: number,
force: boolean = false force: boolean = false,
adjustTime: boolean = true
) { ) {
for (const aid of aids) { for (const aid of aids) {
await scheduleSnapshot(sql, aid, type, targetTime, force); await scheduleSnapshot(sql, aid, type, targetTime, force, adjustTime);
} }
} }

2
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -90,7 +90,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") { if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker"); logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker");
await setSnapshotStatus(sql, id, "no_proxy"); await setSnapshotStatus(sql, id, "no_proxy");
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval); await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true);
return; return;
} else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") { } else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
logger.warn( logger.warn(

23
packages/crawler/mq/exec/takeBulkSnapshot.ts
View File

@ -46,13 +46,13 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
await sql` await sql`
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
VALUES ( VALUES (
${aid}, ${aid},
${views}, ${views},
${danmakus}, ${danmakus},
${replies}, ${replies},
${likes}, ${likes},
${coins}, ${coins},
${shares}, ${shares},
${favorites} ${favorites}
) )
`; `;
@ -74,7 +74,14 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") { if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker"); logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker");
await bulkSetSnapshotStatus(sql, ids, "no_proxy"); await bulkSetSnapshotStatus(sql, ids, "no_proxy");
await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random()); await bulkScheduleSnapshot(
sql,
aidsToFetch,
"normal",
Date.now() + 20 * MINUTE * Math.random(),
false,
true
);
return; return;
} }
logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker"); logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");