alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: don't adjust start time for schedule when no proxy is available

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-06-06 17:11:44 +08:00
parent 1e8d28e194
commit 8cf9395354
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
3 changed files with 22 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
packages/crawler/db/snapshotSchedule.ts
View File

@ -182,7 +182,8 @@ export async function scheduleSnapshot(
aid: number,
type: string,
targetTime: number,
force: boolean = false
force: boolean = false,
adjustTime: boolean = true
) {
let adjustedTime = new Date(targetTime);
const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type);
@ -204,7 +205,7 @@ export async function scheduleSnapshot(
}
}
if (hashActiveSchedule && !force) return;
if (type !== "milestone" && type !== "new") {
if (type !== "milestone" && type !== "new" && adjustTime) {
adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis);
}
logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot");
@ -224,10 +225,11 @@ export async function bulkScheduleSnapshot(
aids: number[],
type: string,
targetTime: number,
force: boolean = false
force: boolean = false,
adjustTime: boolean = true
) {
for (const aid of aids) {
await scheduleSnapshot(sql, aid, type, targetTime, force);
await scheduleSnapshot(sql, aid, type, targetTime, force, adjustTime);
}
}

2
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -90,7 +90,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker");
await setSnapshotStatus(sql, id, "no_proxy");
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval);
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true);
return;
} else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
logger.warn(

23
packages/crawler/mq/exec/takeBulkSnapshot.ts
View File

@ -46,13 +46,13 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
await sql`
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
VALUES (
${aid},
${views},
${danmakus},
${replies},
${likes},
${coins},
${shares},
${aid},
${views},
${danmakus},
${replies},
${likes},
${coins},
${shares},
${favorites}
)
`;
@ -74,7 +74,14 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker");
await bulkSetSnapshotStatus(sql, ids, "no_proxy");
await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random());
await bulkScheduleSnapshot(
sql,
aidsToFetch,
"normal",
Date.now() + 20 * MINUTE * Math.random(),
false,
true
);
return;
}
logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");