diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index d36e172..7def56c 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -182,7 +182,8 @@ export async function scheduleSnapshot( aid: number, type: string, targetTime: number, - force: boolean = false + force: boolean = false, + adjustTime: boolean = true ) { let adjustedTime = new Date(targetTime); const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type); @@ -204,7 +205,7 @@ export async function scheduleSnapshot( } } if (hashActiveSchedule && !force) return; - if (type !== "milestone" && type !== "new") { + if (type !== "milestone" && type !== "new" && adjustTime) { adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis); } logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot"); @@ -224,10 +225,11 @@ export async function bulkScheduleSnapshot( aids: number[], type: string, targetTime: number, - force: boolean = false + force: boolean = false, + adjustTime: boolean = true ) { for (const aid of aids) { - await scheduleSnapshot(sql, aid, type, targetTime, force); + await scheduleSnapshot(sql, aid, type, targetTime, force, adjustTime); } } diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts index 8d8d241..99fc08b 100644 --- a/packages/crawler/mq/exec/snapshotVideo.ts +++ b/packages/crawler/mq/exec/snapshotVideo.ts @@ -90,7 +90,7 @@ export const snapshotVideoWorker = async (job: Job): Promise => { if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") { logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker"); await setSnapshotStatus(sql, id, "no_proxy"); - await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval); + await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true); return; } else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") { logger.warn( diff --git a/packages/crawler/mq/exec/takeBulkSnapshot.ts b/packages/crawler/mq/exec/takeBulkSnapshot.ts index afaf239..00a9354 100644 --- a/packages/crawler/mq/exec/takeBulkSnapshot.ts +++ b/packages/crawler/mq/exec/takeBulkSnapshot.ts @@ -46,13 +46,13 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => { await sql` INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) VALUES ( - ${aid}, - ${views}, - ${danmakus}, - ${replies}, - ${likes}, - ${coins}, - ${shares}, + ${aid}, + ${views}, + ${danmakus}, + ${replies}, + ${likes}, + ${coins}, + ${shares}, ${favorites} ) `; @@ -74,7 +74,14 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => { if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") { logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker"); await bulkSetSnapshotStatus(sql, ids, "no_proxy"); - await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random()); + await bulkScheduleSnapshot( + sql, + aidsToFetch, + "normal", + Date.now() + 20 * MINUTE * Math.random(), + false, + true + ); return; } logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");