alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: prevent videos from being crawled for too long

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-03-17 00:33:28 +08:00
parent a9ac8de547
commit cd8aa826e1
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
2 changed files with 25 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
lib/db/snapshot.ts
View File

@ -71,7 +71,7 @@ export async function getSongSnapshotCount(client: Client, aid: number) {
} }
export async function getShortTermEtaPrediction(client: Client, aid: number) { export async function getShortTermEtaPrediction(client: Client, aid: number) {
const queryResult = await client.queryObject<{eta: number}>( const queryResult = await client.queryObject<{ eta: number }>(
` `
WITH old_snapshot AS ( WITH old_snapshot AS (
SELECT created_at, views SELECT created_at, views
@ -120,6 +120,23 @@ export async function getShortTermEtaPrediction(client: Client, aid: number) {
return queryResult.rows[0].eta; return queryResult.rows[0].eta;
} }
export async function getIntervalFromLastSnapshotToNow(client: Client, aid: number) {
const queryResult = await client.queryObject<{ interval: number }>(
`
SELECT EXTRACT(EPOCH FROM (NOW() - created_at)) AS interval
FROM video_snapshot
WHERE aid = $1
ORDER BY created_at DESC
LIMIT 1;
`,
[aid],
);
if (queryResult.rows.length === 0) {
return null;
}
return queryResult.rows[0].interval;
}
export async function songEligibleForMilestoneSnapshot(client: Client, aid: number) { export async function songEligibleForMilestoneSnapshot(client: Client, aid: number) {
const count = await getSongSnapshotCount(client, aid); const count = await getSongSnapshotCount(client, aid);
if (count < 2) { if (count < 2) {

20
lib/mq/exec/snapshotTick.ts
View File

@ -1,8 +1,9 @@
import { Job } from "bullmq"; import { Job } from "bullmq";
import { MINUTE, SECOND } from "$std/datetime/constants.ts"; import { HOUR, MINUTE, SECOND } from "$std/datetime/constants.ts";
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { db } from "lib/db/init.ts"; import { db } from "lib/db/init.ts";
import { import {
getIntervalFromLastSnapshotToNow,
getShortTermEtaPrediction, getShortTermEtaPrediction,
getSongsNearMilestone, getSongsNearMilestone,
getUnsnapshotedSongs, getUnsnapshotedSongs,
@ -55,19 +56,12 @@ async function processMilestoneSnapshots(client: Client, vidoesNearMilestone: So
let i = 0; let i = 0;
for (const snapshot of vidoesNearMilestone) { for (const snapshot of vidoesNearMilestone) {
if (await snapshotScheduled(snapshot.aid)) { if (await snapshotScheduled(snapshot.aid)) {
logger.silly(
`Video ${snapshot.aid} is already scheduled for snapshot`,
"mq",
"fn:processMilestoneSnapshots",
);
continue; continue;
} }
if (await songEligibleForMilestoneSnapshot(client, snapshot.aid) === false) { const timeFromLastSnapshot = await getIntervalFromLastSnapshotToNow(client, snapshot.aid);
logger.silly( const lastSnapshotLessThan8Hrs = timeFromLastSnapshot && timeFromLastSnapshot * SECOND < 8 * HOUR;
`Video ${snapshot.aid} is not eligible for milestone snapshot`, const notEligible = await songEligibleForMilestoneSnapshot(client, snapshot.aid);
"mq", if (notEligible && lastSnapshotLessThan8Hrs) {
"fn:processMilestoneSnapshots",
);
continue; continue;
} }
const factor = Math.floor(i / 8); const factor = Math.floor(i / 8);
@ -143,7 +137,7 @@ export const takeSnapshotForMilestoneVideoWorker = async (job: Job) => {
eta = viewsToIncrease / (incrementSpeed + DELTA); eta = viewsToIncrease / (incrementSpeed + DELTA);
} }
const scheduledNextSnapshotDelay = eta * SECOND / 3; const scheduledNextSnapshotDelay = eta * SECOND / 3;
const maxInterval = 20 * MINUTE; const maxInterval = 60 * MINUTE;
const minInterval = 1 * SECOND; const minInterval = 1 * SECOND;
const delay = truncate(scheduledNextSnapshotDelay, minInterval, maxInterval); const delay = truncate(scheduledNextSnapshotDelay, minInterval, maxInterval);
await SnapshotQueue.add("snapshotMilestoneVideo", { await SnapshotQueue.add("snapshotMilestoneVideo", {