merge: branch 'main' into ref/structure

packages/crawler/db/snapshot.ts

@@ -7,7 +7,7 @@ export async function getVideosNearMilestone(client: Client) {
         SELECT ls.*
         FROM latest_video_snapshot ls
         WHERE
-            (views >= 90000 AND views < 100000) OR
+            views < 100000 OR
             (views >= 900000 AND views < 1000000) OR
             (views >= 9900000 AND views < 10000000)
     `);

packages/crawler/mq/exec/snapshotTick.ts

@@ -105,7 +105,7 @@ export const collectMilestoneSnapshotsWorker = async (_job: Job) => {
 		for (const video of videos) {
 			const aid = Number(video.aid);
 			const eta = await getAdjustedShortTermETA(client, aid);
-			if (eta > 72) continue;
+			if (eta > 144) continue;
 			const now = Date.now();
 			const scheduledNextSnapshotDelay = eta * HOUR;
 			const maxInterval = 4 * HOUR;
@@ -209,7 +209,7 @@ export const takeSnapshotForVideoWorker = async (job: Job) => {
 		}
 		if (type !== "milestone") return `DONE`;
 		const eta = await getAdjustedShortTermETA(client, aid);
-		if (eta > 72) return "ETA_TOO_LONG";
+		if (eta > 144) return "ETA_TOO_LONG";
 		const now = Date.now();
 		const targetTime = now + eta * HOUR;
 		await scheduleSnapshot(client, aid, type, targetTime);

packages/crawler/mq/scheduling.ts

@@ -1,9 +1,3 @@
-/*
- * Returns the minimum ETA in hours for the next snapshot
- * @param client - Postgres client
- * @param aid - aid of the video
- * @returns ETA in hours
- */
 import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts";
 import { truncate } from "utils/truncate.ts";
 import { closetMilestone } from "./exec/snapshotTick.ts";
@@ -12,6 +6,12 @@ import { HOUR, MINUTE } from "@std/datetime";
 
 const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
 
+/*
+ * Returns the minimum ETA in hours for the next snapshot
+ * @param client - Postgres client
+ * @param aid - aid of the video
+ * @returns ETA in hours
+ */
 export const getAdjustedShortTermETA = async (client: Client, aid: number) => {
 	const latestSnapshot = await getLatestSnapshot(client, aid);
 	// Immediately dispatch a snapshot if there is no snapshot yet