alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: milestone monitoring for videos close to N million views

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-06-06 21:37:55 +08:00
parent 0a6ecc6314
commit b40d24721c
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
4 changed files with 13 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
packages/crawler/db/snapshot.ts
View File

@ -4,20 +4,20 @@ import type { Psql } from "@core/db/psql.d.ts";
export async function getVideosNearMilestone(sql: Psql) {
const queryResult = await sql<LatestSnapshotType[]>`
SELECT ls.*
SELECT ls.*
FROM latest_video_snapshot ls
RIGHT JOIN songs ON songs.aid = ls.aid
RIGHT JOIN songs ON songs.aid = ls.aid
WHERE
(views >= 50000 AND views < 100000) OR
(views >= 900000 AND views < 1000000) OR
(views >= 9900000 AND views < 10000000)
(views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
UNION
SELECT ls.*
FROM latest_video_snapshot ls
WHERE
(views >= 90000 AND views < 100000) OR
(views >= 900000 AND views < 1000000) OR
(views >= 9900000 AND views < 10000000)
(views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
`;
return queryResult.map((row) => {
return {

2
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -84,5 +84,5 @@ export const snapshotTickWorker = async (_job: Job) => {
export const closetMilestone = (views: number) => {
if (views < 100000) return 100000;
if (views < 1000000) return 1000000;
return 10000000;
return Math.ceil(views / 1000000) * 1000000;
};

8
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bullmq";
import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
@ -8,6 +8,7 @@ import { getSongsPublihsedAt } from "db/songs.ts";
import { getAdjustedShortTermETA } from "mq/scheduling.ts";
import { NetSchedulerError } from "@core/net/delegate.ts";
import { sql } from "@core/db/dbNew.ts";
import { closetMilestone } from "./snapshotTick.ts";
const snapshotTypeToTaskMap: { [key: string]: string } = {
milestone: "snapshotMilestoneVideo",
@ -21,6 +22,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
const type = job.data.type;
const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo";
const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE;
const latestSnapshot = await getLatestSnapshot(sql, aid);
try {
const exists = await snapshotScheduleExists(sql, id);
if (!exists) {
@ -71,6 +73,10 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true);
}
if (type !== "milestone") return;
const alreadyAchievedMilestone = stat.views > closetMilestone(latestSnapshot.views);
if (alreadyAchievedMilestone) {
return;
}
const eta = await getAdjustedShortTermETA(sql, aid);
if (eta > 144) {
const etaHoursString = eta.toFixed(2) + " hrs";

2
packages/crawler/package.json
View File

@ -1,6 +1,6 @@
{
"name": "crawler",
"version": "1.2.26",
"version": "1.3.0",
"scripts": {
"test": "bun --env-file=.env.test run vitest",
"worker:main": "bun run ./src/worker.ts",