From b40d24721cf26646611f9567ab7670ededd6bf77 Mon Sep 17 00:00:00 2001
From: alikia2x <alikia2x@outlook.com>
Date: Fri, 6 Jun 2025 21:37:55 +0800
Subject: [PATCH] add: milestone monitoring for videos close to N million views

---
 packages/crawler/db/snapshot.ts           | 8 ++++----
 packages/crawler/mq/exec/snapshotTick.ts  | 2 +-
 packages/crawler/mq/exec/snapshotVideo.ts | 8 +++++++-
 packages/crawler/package.json             | 2 +-
 4 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts
index 0b4c5fe..e2c8555 100644
--- a/packages/crawler/db/snapshot.ts
+++ b/packages/crawler/db/snapshot.ts
@@ -4,20 +4,20 @@ import type { Psql } from "@core/db/psql.d.ts";
 
 export async function getVideosNearMilestone(sql: Psql) {
 	const queryResult = await sql<LatestSnapshotType[]>`
-        SELECT ls.*
+    	SELECT ls.*
         FROM latest_video_snapshot ls
-                 RIGHT JOIN songs ON songs.aid = ls.aid
+                RIGHT JOIN songs ON songs.aid = ls.aid
         WHERE
             (views >= 50000 AND views < 100000) OR
             (views >= 900000 AND views < 1000000) OR
-            (views >= 9900000 AND views < 10000000)
+            (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
         UNION
         SELECT ls.*
         FROM latest_video_snapshot ls
         WHERE
             (views >= 90000 AND views < 100000) OR
             (views >= 900000 AND views < 1000000) OR
-            (views >= 9900000 AND views < 10000000)
+            (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
     `;
 	return queryResult.map((row) => {
 		return {
diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts
index ca596c8..784bb04 100644
--- a/packages/crawler/mq/exec/snapshotTick.ts
+++ b/packages/crawler/mq/exec/snapshotTick.ts
@@ -84,5 +84,5 @@ export const snapshotTickWorker = async (_job: Job) => {
 export const closetMilestone = (views: number) => {
 	if (views < 100000) return 100000;
 	if (views < 1000000) return 1000000;
-	return 10000000;
+	return Math.ceil(views / 1000000) * 1000000;
 };
diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts
index 99fc08b..9f806d5 100644
--- a/packages/crawler/mq/exec/snapshotVideo.ts
+++ b/packages/crawler/mq/exec/snapshotVideo.ts
@@ -1,5 +1,5 @@
 import { Job } from "bullmq";
-import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
+import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
 import logger from "@core/log/logger.ts";
 import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
 import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
@@ -8,6 +8,7 @@ import { getSongsPublihsedAt } from "db/songs.ts";
 import { getAdjustedShortTermETA } from "mq/scheduling.ts";
 import { NetSchedulerError } from "@core/net/delegate.ts";
 import { sql } from "@core/db/dbNew.ts";
+import { closetMilestone } from "./snapshotTick.ts";
 
 const snapshotTypeToTaskMap: { [key: string]: string } = {
 	milestone: "snapshotMilestoneVideo",
@@ -21,6 +22,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 	const type = job.data.type;
 	const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo";
 	const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE;
+	const latestSnapshot = await getLatestSnapshot(sql, aid);
 	try {
 		const exists = await snapshotScheduleExists(sql, id);
 		if (!exists) {
@@ -71,6 +73,10 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
 			await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true);
 		}
 		if (type !== "milestone") return;
+		const alreadyAchievedMilestone = stat.views > closetMilestone(latestSnapshot.views);
+		if (alreadyAchievedMilestone) {
+			return;
+		}
 		const eta = await getAdjustedShortTermETA(sql, aid);
 		if (eta > 144) {
 			const etaHoursString = eta.toFixed(2) + " hrs";
diff --git a/packages/crawler/package.json b/packages/crawler/package.json
index 4302fdc..cf0b6b0 100644
--- a/packages/crawler/package.json
+++ b/packages/crawler/package.json
@@ -1,6 +1,6 @@
 {
 	"name": "crawler",
-	"version": "1.2.26",
+	"version": "1.3.0",
 	"scripts": {
 		"test": "bun --env-file=.env.test run vitest",
 		"worker:main": "bun run ./src/worker.ts",