diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index 70bed97..119e353 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -7,7 +7,7 @@ export async function getVideosNearMilestone(client: Client) { SELECT ls.* FROM latest_video_snapshot ls WHERE - (views >= 90000 AND views < 100000) OR + views < 100000 OR (views >= 900000 AND views < 1000000) OR (views >= 9900000 AND views < 10000000) `); diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts index 6b8365a..31f2776 100644 --- a/packages/crawler/mq/exec/snapshotTick.ts +++ b/packages/crawler/mq/exec/snapshotTick.ts @@ -105,7 +105,7 @@ export const collectMilestoneSnapshotsWorker = async (_job: Job) => { for (const video of videos) { const aid = Number(video.aid); const eta = await getAdjustedShortTermETA(client, aid); - if (eta > 72) continue; + if (eta > 144) continue; const now = Date.now(); const scheduledNextSnapshotDelay = eta * HOUR; const maxInterval = 4 * HOUR; @@ -209,7 +209,7 @@ export const takeSnapshotForVideoWorker = async (job: Job) => { } if (type !== "milestone") return `DONE`; const eta = await getAdjustedShortTermETA(client, aid); - if (eta > 72) return "ETA_TOO_LONG"; + if (eta > 144) return "ETA_TOO_LONG"; const now = Date.now(); const targetTime = now + eta * HOUR; await scheduleSnapshot(client, aid, type, targetTime); diff --git a/packages/crawler/mq/scheduling.ts b/packages/crawler/mq/scheduling.ts index 554874d..e2451dd 100644 --- a/packages/crawler/mq/scheduling.ts +++ b/packages/crawler/mq/scheduling.ts @@ -1,9 +1,3 @@ -/* - * Returns the minimum ETA in hours for the next snapshot - * @param client - Postgres client - * @param aid - aid of the video - * @returns ETA in hours - */ import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts"; import { truncate } from "utils/truncate.ts"; import { closetMilestone } from "./exec/snapshotTick.ts"; @@ -12,6 +6,12 @@ import { HOUR, MINUTE } from "@std/datetime"; const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); +/* + * Returns the minimum ETA in hours for the next snapshot + * @param client - Postgres client + * @param aid - aid of the video + * @returns ETA in hours + */ export const getAdjustedShortTermETA = async (client: Client, aid: number) => { const latestSnapshot = await getLatestSnapshot(client, aid); // Immediately dispatch a snapshot if there is no snapshot yet