alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ref: the imports of db schema

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-04-05 18:26:01 +08:00
parent f39fef0d9a
commit 33c6a3c1f8
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
17 changed files with 45 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.idea/cvsa.iml
View File

@ -14,6 +14,9 @@
<excludeFolder url="file://$MODULE_DIR$/logs" />
<excludeFolder url="file://$MODULE_DIR$/model" />
<excludeFolder url="file://$MODULE_DIR$/src/db" />
<excludeFolder url="file://$MODULE_DIR$/.idea" />
<excludeFolder url="file://$MODULE_DIR$/.vscode" />
<excludeFolder url="file://$MODULE_DIR$/.zed" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />

6
.idea/deno.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DenoSettings">
<option name="useDenoValue" value="ENABLE" />
</component>
</project>

1
deno.json
View File

@ -16,7 +16,6 @@
"imports": {
"@astrojs/node": "npm:@astrojs/node@^9.1.3",
"@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8",
"@core/db/": "./packages/core/db/",
"date-fns": "npm:date-fns@^4.1.0"
}
}

3
packages/backend/deno.json
View File

@ -4,7 +4,8 @@
"@rabbit-company/argon2id": "jsr:@rabbit-company/argon2id@^2.1.0",
"hono": "jsr:@hono/hono@^4.7.5",
"zod": "npm:zod",
"yup": "npm:yup"
"yup": "npm:yup",
"@core/": "../core/"
},
"tasks": {
"dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts",

2
packages/core/db/videoSnapshot.ts
View File

@ -1,5 +1,5 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { VideoSnapshotType } from "@core/db/schema.d.ts";
import { VideoSnapshotType } from "./schema.d.ts";
export async function getVideoSnapshots(client: Client, aid: number, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;

4
packages/core/deno.json
View File

@ -0,0 +1,4 @@
{
"name": "@cvsa/core",
"exports": "./main.ts"
}

1
packages/core/main.ts Normal file
View File

@ -0,0 +1 @@
export const DB_VERSION = 10;

2
packages/crawler/db/allData.ts
View File

@ -1,5 +1,5 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { AllDataType, BiliUserType } from "@core/db/schema.d.ts";
import { AllDataType, BiliUserType } from "@core/db/schema";
import Akari from "ml/akari.ts";
export async function videoExistsInAllData(client: Client, aid: number) {

5
packages/crawler/db/snapshot.ts
View File

@ -1,5 +1,6 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { LatestSnapshotType } from "@core/db/schema.d.ts";
import { LatestSnapshotType } from "@core/db/schema";
import { SnapshotNumber } from "mq/task/getVideoStats.ts";
export async function getVideosNearMilestone(client: Client) {
const queryResult = await client.queryObject<LatestSnapshotType>(`
@ -18,7 +19,7 @@ export async function getVideosNearMilestone(client: Client) {
});
}
export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | LatestSnapshotType> {
export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | SnapshotNumber> {
const queryResult = await client.queryObject<LatestSnapshotType>(
`
SELECT *

26
packages/crawler/db/snapshotSchedule.ts
View File

@ -1,6 +1,5 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts";
import { SnapshotScheduleType } from "@core/db/schema.d.ts";
import { SnapshotScheduleType } from "@core/db/schema";
import logger from "log/logger.ts";
import { MINUTE } from "$std/datetime/constants.ts";
import { redis } from "db/redis.ts";
@ -11,8 +10,7 @@ const REDIS_KEY = "cvsa:snapshot_window_counts";
function getCurrentWindowIndex(): number {
const now = new Date();
const minutesSinceMidnight = now.getHours() * 60 + now.getMinutes();
const currentWindow = Math.floor(minutesSinceMidnight / 5);
return currentWindow;
return Math.floor(minutesSinceMidnight / 5);
}
export async function refreshSnapshotWindowCounts(client: Client, redisClient: Redis) {
@ -162,24 +160,6 @@ export async function getLatestSnapshot(client: Client, aid: number): Promise<Sn
};
}
/*
* Returns the number of snapshot schedules within the specified range.
* @param client The database client.
* @param start The start time of the range. (Timestamp in milliseconds)
* @param end The end time of the range. (Timestamp in milliseconds)
*/
export async function getSnapshotScheduleCountWithinRange(client: Client, start: number, end: number) {
const startTimeString = formatTimestampToPsql(start);
const endTimeString = formatTimestampToPsql(end);
const query = `
SELECT COUNT(*) FROM snapshot_schedule
WHERE started_at BETWEEN $1 AND $2
AND status = 'pending'
`;
const res = await client.queryObject<{ count: number }>(query, [startTimeString, endTimeString]);
return res.rows[0].count;
}
/*
* Creates a new snapshot schedule record.
* @param client The database client.
@ -227,7 +207,7 @@ export async function adjustSnapshotTime(
const initialOffset = currentWindow + Math.max(targetOffset, 0);
let timePerIteration = 0;
let timePerIteration: number;
const MAX_ITERATIONS = 2880;
let iters = 0;
const t = performance.now();

3
packages/crawler/deno.json
View File

@ -37,7 +37,8 @@
"express": "npm:express",
"src/": "./src/",
"onnxruntime": "npm:onnxruntime-node@1.19.2",
"chalk": "npm:chalk"
"chalk": "npm:chalk",
"@core/db/schema": "../core/db/schema.d.ts"
},
"exports": "./main.ts"
}

2
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -216,7 +216,7 @@ export const regularSnapshotsWorker = async (_job: Job) => {
} catch (e) {
logger.error(e as Error, "mq", "fn:regularSnapshotsWorker");
} finally {
lockManager.releaseLock("dispatchRegularSnapshots");
await lockManager.releaseLock("dispatchRegularSnapshots");
client.release();
}
};

3
packages/crawler/mq/scheduler.ts
View File

@ -217,8 +217,7 @@ class NetScheduler {
const providerLimiterId = "provider-" + proxyName + "-" + provider;
if (!this.proxyLimiters[proxyLimiterId]) {
const providerLimiter = this.providerLimiters[providerLimiterId];
const providerAvailable = await providerLimiter.getAvailability();
return providerAvailable;
return await providerLimiter.getAvailability();
}
const proxyLimiter = this.proxyLimiters[proxyLimiterId];
const providerLimiter = this.providerLimiters[providerLimiterId];

19
packages/crawler/mq/task/getVideoStats.ts
View File

@ -1,8 +1,19 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { getVideoInfo } from "net/getVideoInfo.ts";
import { LatestSnapshotType } from "@core/db/schema.d.ts";
import logger from "log/logger.ts";
export interface SnapshotNumber {
time: number;
views: number;
coins: number;
likes: number;
favorites: number;
shares: number;
danmakus: number;
aid: number;
replies: number;
}
/*
* Fetch video stats from bilibili API and insert into database
* @returns {Promise<number|VideoSnapshot>}
@ -17,7 +28,7 @@ export async function insertVideoSnapshot(
client: Client,
aid: number,
task: string,
): Promise<number | LatestSnapshotType> {
): Promise<number | SnapshotNumber> {
const data = await getVideoInfo(aid, task);
if (typeof data == "number") {
return data;
@ -42,7 +53,7 @@ export async function insertVideoSnapshot(
logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
const snapshot: LatestSnapshotType = {
return {
aid,
views,
danmakus,
@ -53,6 +64,4 @@ export async function insertVideoSnapshot(
favorites,
time,
};
return snapshot;
}

3
packages/crawler/net/bulkGetVideoStats.ts
View File

@ -12,8 +12,7 @@ import logger from "log/logger.ts";
* - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
*/
export async function bulkGetVideoStats(aids: number[]): Promise<MediaListInfoData | number> {
const baseURL = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
let url = baseURL;
let url = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
for (const aid of aids) {
url += `${aid}:2,`;
}

2
packages/crawler/src/filterWorker.ts
View File

@ -18,7 +18,7 @@ Deno.addSignalListener("SIGTERM", async () => {
Deno.exit();
});
Akari.init();
await Akari.init();
const filterWorker = new Worker(
"classifyVideo",