alikia2x/cvsa
1
0
Fork 0
Code

update: the rate limit config

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-10-29 22:49:21 +08:00
parent 093bc3697c
commit 55535a16ce
2 changed files with 26 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
packages/core/net/delegate.ts
View File

@ -310,33 +310,27 @@ const videoInfoRateLimiterConfig: RateLimiterConfig[] = [
const biliLimiterConfig: RateLimiterConfig[] = [
{
duration: 1,
max: 6
},
{
duration: 5,
max: 20
},
{
duration: 30,
max: 100
duration: 15,
max: 130
},
{
duration: 5 * 60,
max: 500
max: 2000
}
];
const bili_test = [...biliLimiterConfig];
bili_test[0].max = 10;
bili_test[1].max = 36;
bili_test[2].max = 150;
bili_test[3].max = 1000;
const bili_normal = [...biliLimiterConfig];
bili_normal[0].max = 5;
bili_normal[1].max = 40;
bili_normal[2].max = 200;
const bili_strict = [...biliLimiterConfig];
bili_strict[0].max = 1;
bili_strict[1].max = 4;
bili_strict[2].max = 12;
bili_strict[3].max = 100;
bili_strict[1].max = 6;
bili_strict[2].max = 100;
/*
Execution order for setup:
@ -362,52 +356,35 @@ but both should come after addProxy and addTask to ensure proper setup and depen
*/
const regions = [
"shanghai",
"hangzhou",
"qingdao",
"beijing",
"hangzhou",
"shanghai",
"qingdao",
"zhangjiakou",
"chengdu",
"shenzhen",
"huhehaote"
];
const fcProxies = regions.map((region) => `alicloud-${region}`);
const fcProxiesL = regions.slice(2).map((region) => `alicloud-${region}`);
networkDelegate.addProxy("native", "native", "");
for (const region of regions) {
networkDelegate.addProxy(`alicloud-${region}`, "alicloud-fc", region);
}
networkDelegate.addTask("test", "test", "all");
networkDelegate.addTask("getVideoInfo", "bilibili", "all");
networkDelegate.addTask("getLatestVideos", "bilibili", "all");
networkDelegate.addTask(
"snapshotMilestoneVideo",
"bilibili",
regions.map((region) => `alicloud-${region}`)
);
networkDelegate.addTask("snapshotVideo", "bili_test", [
"alicloud-qingdao",
"alicloud-shanghai",
"alicloud-zhangjiakou",
"alicloud-chengdu",
"alicloud-shenzhen",
"alicloud-huhehaote"
]);
networkDelegate.addTask("bulkSnapshot", "bili_strict", [
"alicloud-qingdao",
"alicloud-shanghai",
"alicloud-zhangjiakou",
"alicloud-chengdu",
"alicloud-shenzhen",
"alicloud-huhehaote"
]);
networkDelegate.addTask("snapshotMilestoneVideo", "bilibili", fcProxies);
networkDelegate.addTask("snapshotVideo", "bilibili", fcProxiesL);
networkDelegate.addTask("bulkSnapshot", "bilibili", fcProxiesL);
networkDelegate.setTaskLimiter("getVideoInfo", videoInfoRateLimiterConfig);
networkDelegate.setTaskLimiter("getLatestVideos", null);
networkDelegate.setTaskLimiter("snapshotMilestoneVideo", null);
networkDelegate.setTaskLimiter("snapshotVideo", null);
networkDelegate.setTaskLimiter("bulkSnapshot", null);
networkDelegate.setTaskLimiter("test", null);
networkDelegate.setTaskLimiter("bulkSnapshot", bili_strict);
networkDelegate.setTaskLimiter("getLatestVideos", bili_strict);
networkDelegate.setTaskLimiter("getVideoInfo", bili_strict);
networkDelegate.setTaskLimiter("snapshotVideo", bili_normal);
networkDelegate.setProviderLimiter("test", []);
networkDelegate.setProviderLimiter("bilibili", biliLimiterConfig);
networkDelegate.setProviderLimiter("bili_test", bili_test);
networkDelegate.setProviderLimiter("bili_strict", bili_strict);
export default networkDelegate;

6
packages/crawler/db/snapshotSchedule.ts
View File

@ -24,7 +24,7 @@ export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: Redis)
(EXTRACT(minute FROM started_at)::int / 5 * INTERVAL '5 minutes') AS window_start,
COUNT(*) AS count
FROM snapshot_schedule
WHERE started_at >= NOW() AND status = 'pending' AND started_at <= NOW() + INTERVAL '10 days'
WHERE started_at >= NOW() AND status = 'pending' AND started_at <= NOW() + INTERVAL '14 days'
GROUP BY 1
ORDER BY window_start
`;
@ -209,7 +209,7 @@ export async function scheduleSnapshot(
}
if (hasActiveSchedule && !force) return;
if (type !== "milestone" && type !== "new" && adjustTime) {
adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis);
adjustedTime = await adjustSnapshotTime(new Date(targetTime), 3000, redis);
}
logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot");
return sql`
@ -246,7 +246,7 @@ export async function adjustSnapshotTime(
const initialOffset = currentWindow + Math.max(targetOffset, 0);
const MAX_ITERATIONS = 2880;
const MAX_ITERATIONS = 4032; // 10 days
for (let i = initialOffset; i < MAX_ITERATIONS; i++) {
const offset = i;
const count = await getWindowCount(redisClient, offset);