alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: milestone monitoring for videos close to N million views

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-06-06 21:37:55 +08:00
parent 0a6ecc6314
commit b40d24721c
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
4 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/crawler/db/snapshot.ts
View File

@ -10,14 +10,14 @@ export async function getVideosNearMilestone(sql: Psql) {
WHERE WHERE
(views >= 50000 AND views < 100000) OR (views >= 50000 AND views < 100000) OR
(views >= 900000 AND views < 1000000) OR (views >= 900000 AND views < 1000000) OR
(views >= 9900000 AND views < 10000000) (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
UNION UNION
SELECT ls.* SELECT ls.*
FROM latest_video_snapshot ls FROM latest_video_snapshot ls
WHERE WHERE
(views >= 90000 AND views < 100000) OR (views >= 90000 AND views < 100000) OR
(views >= 900000 AND views < 1000000) OR (views >= 900000 AND views < 1000000) OR
(views >= 9900000 AND views < 10000000) (views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
`; `;
return queryResult.map((row) => { return queryResult.map((row) => {
return { return {

2
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -84,5 +84,5 @@ export const snapshotTickWorker = async (_job: Job) => {
export const closetMilestone = (views: number) => { export const closetMilestone = (views: number) => {
if (views < 100000) return 100000; if (views < 100000) return 100000;
if (views < 1000000) return 1000000; if (views < 1000000) return 1000000;
return 10000000; return Math.ceil(views / 1000000) * 1000000;
}; };

8
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bullmq"; import { Job } from "bullmq";
import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts"; import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts"; import logger from "@core/log/logger.ts";
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts"; import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
@ -8,6 +8,7 @@ import { getSongsPublihsedAt } from "db/songs.ts";
import { getAdjustedShortTermETA } from "mq/scheduling.ts"; import { getAdjustedShortTermETA } from "mq/scheduling.ts";
import { NetSchedulerError } from "@core/net/delegate.ts"; import { NetSchedulerError } from "@core/net/delegate.ts";
import { sql } from "@core/db/dbNew.ts"; import { sql } from "@core/db/dbNew.ts";
import { closetMilestone } from "./snapshotTick.ts";
const snapshotTypeToTaskMap: { [key: string]: string } = { const snapshotTypeToTaskMap: { [key: string]: string } = {
milestone: "snapshotMilestoneVideo", milestone: "snapshotMilestoneVideo",
@ -21,6 +22,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
const type = job.data.type; const type = job.data.type;
const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo"; const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo";
const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE; const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE;
const latestSnapshot = await getLatestSnapshot(sql, aid);
try { try {
const exists = await snapshotScheduleExists(sql, id); const exists = await snapshotScheduleExists(sql, id);
if (!exists) { if (!exists) {
@ -71,6 +73,10 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true); await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true);
} }
if (type !== "milestone") return; if (type !== "milestone") return;
const alreadyAchievedMilestone = stat.views > closetMilestone(latestSnapshot.views);
if (alreadyAchievedMilestone) {
return;
}
const eta = await getAdjustedShortTermETA(sql, aid); const eta = await getAdjustedShortTermETA(sql, aid);
if (eta > 144) { if (eta > 144) {
const etaHoursString = eta.toFixed(2) + " hrs"; const etaHoursString = eta.toFixed(2) + " hrs";

2
packages/crawler/package.json
View File

@ -1,6 +1,6 @@
{ {
"name": "crawler", "name": "crawler",
"version": "1.2.26", "version": "1.3.0",
"scripts": { "scripts": {
"test": "bun --env-file=.env.test run vitest", "test": "bun --env-file=.env.test run vitest",
"worker:main": "bun run ./src/worker.ts", "worker:main": "bun run ./src/worker.ts",