alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: expand the candidate range of milestone snapshot

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-04-13 16:50:19 +08:00
parent b5dbf293a2
commit f4d08e944a
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
3 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
packages/crawler/db/snapshot.ts
View File

@ -7,7 +7,7 @@ export async function getVideosNearMilestone(client: Client) {
SELECT ls.* SELECT ls.*
FROM latest_video_snapshot ls FROM latest_video_snapshot ls
WHERE WHERE
(views >= 90000 AND views < 100000) OR views < 100000 OR
(views >= 900000 AND views < 1000000) OR (views >= 900000 AND views < 1000000) OR
(views >= 9900000 AND views < 10000000) (views >= 9900000 AND views < 10000000)
`); `);

4
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -110,7 +110,7 @@ export const collectMilestoneSnapshotsWorker = async (_job: Job) => {
for (const video of videos) { for (const video of videos) {
const aid = Number(video.aid); const aid = Number(video.aid);
const eta = await getAdjustedShortTermETA(client, aid); const eta = await getAdjustedShortTermETA(client, aid);
if (eta > 72) continue; if (eta > 144) continue;
const now = Date.now(); const now = Date.now();
const scheduledNextSnapshotDelay = eta * HOUR; const scheduledNextSnapshotDelay = eta * HOUR;
const maxInterval = 4 * HOUR; const maxInterval = 4 * HOUR;
@ -299,7 +299,7 @@ export const takeSnapshotForVideoWorker = async (job: Job) => {
} }
if (type !== "milestone") return `DONE`; if (type !== "milestone") return `DONE`;
const eta = await getAdjustedShortTermETA(client, aid); const eta = await getAdjustedShortTermETA(client, aid);
if (eta > 72) return "ETA_TOO_LONG"; if (eta > 144) return "ETA_TOO_LONG";
const now = Date.now(); const now = Date.now();
const targetTime = now + eta * HOUR; const targetTime = now + eta * HOUR;
await scheduleSnapshot(client, aid, type, targetTime); await scheduleSnapshot(client, aid, type, targetTime);

12
packages/crawler/mq/scheduling.ts
View File

@ -1,9 +1,3 @@
/*
* Returns the minimum ETA in hours for the next snapshot
* @param client - Postgres client
* @param aid - aid of the video
* @returns ETA in hours
*/
import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts"; import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts";
import { truncate } from "utils/truncate.ts"; import { truncate } from "utils/truncate.ts";
import { closetMilestone } from "./exec/snapshotTick.ts"; import { closetMilestone } from "./exec/snapshotTick.ts";
@ -12,6 +6,12 @@ import { HOUR, MINUTE } from "$std/datetime/constants.ts";
const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
/*
* Returns the minimum ETA in hours for the next snapshot
* @param client - Postgres client
* @param aid - aid of the video
* @returns ETA in hours
*/
export const getAdjustedShortTermETA = async (client: Client, aid: number) => { export const getAdjustedShortTermETA = async (client: Client, aid: number) => {
const latestSnapshot = await getLatestSnapshot(client, aid); const latestSnapshot = await getLatestSnapshot(client, aid);
// Immediately dispatch a snapshot if there is no snapshot yet // Immediately dispatch a snapshot if there is no snapshot yet