From f4d08e944ab530d4e3debbbb36c72544ff4bf10a Mon Sep 17 00:00:00 2001 From: alikia2x Date: Sun, 13 Apr 2025 16:50:19 +0800 Subject: [PATCH] fix: expand the candidate range of milestone snapshot --- packages/crawler/db/snapshot.ts | 2 +- packages/crawler/mq/exec/snapshotTick.ts | 4 ++-- packages/crawler/mq/scheduling.ts | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index 70bed97..119e353 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -7,7 +7,7 @@ export async function getVideosNearMilestone(client: Client) { SELECT ls.* FROM latest_video_snapshot ls WHERE - (views >= 90000 AND views < 100000) OR + views < 100000 OR (views >= 900000 AND views < 1000000) OR (views >= 9900000 AND views < 10000000) `); diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts index 1c15bc4..ac7f04e 100644 --- a/packages/crawler/mq/exec/snapshotTick.ts +++ b/packages/crawler/mq/exec/snapshotTick.ts @@ -110,7 +110,7 @@ export const collectMilestoneSnapshotsWorker = async (_job: Job) => { for (const video of videos) { const aid = Number(video.aid); const eta = await getAdjustedShortTermETA(client, aid); - if (eta > 72) continue; + if (eta > 144) continue; const now = Date.now(); const scheduledNextSnapshotDelay = eta * HOUR; const maxInterval = 4 * HOUR; @@ -299,7 +299,7 @@ export const takeSnapshotForVideoWorker = async (job: Job) => { } if (type !== "milestone") return `DONE`; const eta = await getAdjustedShortTermETA(client, aid); - if (eta > 72) return "ETA_TOO_LONG"; + if (eta > 144) return "ETA_TOO_LONG"; const now = Date.now(); const targetTime = now + eta * HOUR; await scheduleSnapshot(client, aid, type, targetTime); diff --git a/packages/crawler/mq/scheduling.ts b/packages/crawler/mq/scheduling.ts index e2f59e0..e490787 100644 --- a/packages/crawler/mq/scheduling.ts +++ b/packages/crawler/mq/scheduling.ts @@ -1,9 +1,3 @@ -/* - * Returns the minimum ETA in hours for the next snapshot - * @param client - Postgres client - * @param aid - aid of the video - * @returns ETA in hours - */ import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts"; import { truncate } from "utils/truncate.ts"; import { closetMilestone } from "./exec/snapshotTick.ts"; @@ -12,6 +6,12 @@ import { HOUR, MINUTE } from "$std/datetime/constants.ts"; const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); +/* + * Returns the minimum ETA in hours for the next snapshot + * @param client - Postgres client + * @param aid - aid of the video + * @returns ETA in hours + */ export const getAdjustedShortTermETA = async (client: Client, aid: number) => { const latestSnapshot = await getLatestSnapshot(client, aid); // Immediately dispatch a snapshot if there is no snapshot yet