ref: the imports of db schema

.idea/cvsa.iml

@@ -14,6 +14,9 @@
       <excludeFolder url="file://$MODULE_DIR$/logs" />
       <excludeFolder url="file://$MODULE_DIR$/model" />
       <excludeFolder url="file://$MODULE_DIR$/src/db" />
+      <excludeFolder url="file://$MODULE_DIR$/.idea" />
+      <excludeFolder url="file://$MODULE_DIR$/.vscode" />
+      <excludeFolder url="file://$MODULE_DIR$/.zed" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

.idea/deno.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DenoSettings">
+    <option name="useDenoValue" value="ENABLE" />
+  </component>
+</project>

deno.json

@@ -16,7 +16,6 @@
 	"imports": {
 		"@astrojs/node": "npm:@astrojs/node@^9.1.3",
 		"@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8",
-		"@core/db/": "./packages/core/db/",
 		"date-fns": "npm:date-fns@^4.1.0"
 	}
 }

packages/backend/deno.json

@@ -4,7 +4,8 @@
 		"@rabbit-company/argon2id": "jsr:@rabbit-company/argon2id@^2.1.0",
 		"hono": "jsr:@hono/hono@^4.7.5",
 		"zod": "npm:zod",
-		"yup": "npm:yup"
+		"yup": "npm:yup",
+		"@core/": "../core/"
 	},
 	"tasks": {
 		"dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts",

packages/core/db/videoSnapshot.ts

@@ -1,5 +1,5 @@
 import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
-import { VideoSnapshotType } from "@core/db/schema.d.ts";
+import { VideoSnapshotType } from "./schema.d.ts";
 
 export async function getVideoSnapshots(client: Client, aid: number, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
     const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;

packages/core/deno.json

@@ -0,0 +1,4 @@
+{
+	"name": "@cvsa/core",
+	"exports": "./main.ts"
+}

packages/core/main.ts

@@ -0,0 +1 @@
+export const DB_VERSION = 10;

packages/crawler/db/allData.ts

@@ -1,5 +1,5 @@
 import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
-import { AllDataType, BiliUserType } from "@core/db/schema.d.ts";
+import { AllDataType, BiliUserType } from "@core/db/schema";
 import Akari from "ml/akari.ts";
 
 export async function videoExistsInAllData(client: Client, aid: number) {

packages/crawler/db/snapshot.ts

@@ -1,5 +1,6 @@
 import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
-import { LatestSnapshotType } from "@core/db/schema.d.ts";
+import { LatestSnapshotType } from "@core/db/schema";
+import { SnapshotNumber } from "mq/task/getVideoStats.ts";
 
 export async function getVideosNearMilestone(client: Client) {
 	const queryResult = await client.queryObject<LatestSnapshotType>(`
@@ -18,7 +19,7 @@ export async function getVideosNearMilestone(client: Client) {
 	});
 }
 
-export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | LatestSnapshotType> {
+export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | SnapshotNumber> {
 	const queryResult = await client.queryObject<LatestSnapshotType>(
 		`
 	    SELECT *

packages/crawler/db/snapshotSchedule.ts

@@ -1,6 +1,5 @@
 import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
-import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts";
-import { SnapshotScheduleType } from "@core/db/schema.d.ts";
+import { SnapshotScheduleType } from "@core/db/schema";
 import logger from "log/logger.ts";
 import { MINUTE } from "$std/datetime/constants.ts";
 import { redis } from "db/redis.ts";
@@ -11,8 +10,7 @@ const REDIS_KEY = "cvsa:snapshot_window_counts";
 function getCurrentWindowIndex(): number {
 	const now = new Date();
 	const minutesSinceMidnight = now.getHours() * 60 + now.getMinutes();
-	const currentWindow = Math.floor(minutesSinceMidnight / 5);
-	return currentWindow;
+	return Math.floor(minutesSinceMidnight / 5);
 }
 
 export async function refreshSnapshotWindowCounts(client: Client, redisClient: Redis) {
@@ -162,24 +160,6 @@ export async function getLatestSnapshot(client: Client, aid: number): Promise<Sn
 	};
 }
 
-/*
- * Returns the number of snapshot schedules within the specified range.
- * @param client The database client.
- * @param start The start time of the range. (Timestamp in milliseconds)
- * @param end The end time of the range. (Timestamp in milliseconds)
- */
-export async function getSnapshotScheduleCountWithinRange(client: Client, start: number, end: number) {
-	const startTimeString = formatTimestampToPsql(start);
-	const endTimeString = formatTimestampToPsql(end);
-	const query = `
-		SELECT COUNT(*) FROM snapshot_schedule
-		WHERE started_at BETWEEN $1 AND $2
-		AND status = 'pending'
-	`;
-	const res = await client.queryObject<{ count: number }>(query, [startTimeString, endTimeString]);
-	return res.rows[0].count;
-}
-
 /*
  * Creates a new snapshot schedule record.
  * @param client The database client.
@@ -227,7 +207,7 @@ export async function adjustSnapshotTime(
 
 	const initialOffset = currentWindow + Math.max(targetOffset, 0);
 
-	let timePerIteration = 0;
+	let timePerIteration: number;
 	const MAX_ITERATIONS = 2880;
 	let iters = 0;
 	const t = performance.now();

packages/crawler/deno.json

@@ -37,7 +37,8 @@
 		"express": "npm:express",
 		"src/": "./src/",
 		"onnxruntime": "npm:onnxruntime-node@1.19.2",
-		"chalk": "npm:chalk"
+		"chalk": "npm:chalk",
+		"@core/db/schema": "../core/db/schema.d.ts"
 	},
 	"exports": "./main.ts"
 }

packages/crawler/mq/exec/snapshotTick.ts

@@ -216,7 +216,7 @@ export const regularSnapshotsWorker = async (_job: Job) => {
 	} catch (e) {
 		logger.error(e as Error, "mq", "fn:regularSnapshotsWorker");
 	} finally {
-		lockManager.releaseLock("dispatchRegularSnapshots");
+		await lockManager.releaseLock("dispatchRegularSnapshots");
 		client.release();
 	}
 };

packages/crawler/mq/scheduler.ts

@@ -217,8 +217,7 @@ class NetScheduler {
 			const providerLimiterId = "provider-" + proxyName + "-" + provider;
 			if (!this.proxyLimiters[proxyLimiterId]) {
 				const providerLimiter = this.providerLimiters[providerLimiterId];
-				const providerAvailable = await providerLimiter.getAvailability();
-				return providerAvailable;
+				return await providerLimiter.getAvailability();
 			}
 			const proxyLimiter = this.proxyLimiters[proxyLimiterId];
 			const providerLimiter = this.providerLimiters[providerLimiterId];

packages/crawler/mq/task/getVideoDetails.ts

@@ -53,7 +53,7 @@ export async function insertVideoInfo(client: Client, aid: number) {
 		query,
 		[aid, stat.view, stat.danmaku, stat.reply, stat.like, stat.coin, stat.share, stat.favorite],
 	);
-	
+
 	logger.log(`Inserted video metadata for aid: ${aid}`, "mq");
 	await ClassifyVideoQueue.add("classifyVideo", { aid });
 }

packages/crawler/mq/task/getVideoStats.ts

@@ -1,8 +1,19 @@
 import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
 import { getVideoInfo } from "net/getVideoInfo.ts";
-import { LatestSnapshotType } from "@core/db/schema.d.ts";
 import logger from "log/logger.ts";
 
+export interface SnapshotNumber {
+	time: number;
+	views: number;
+	coins: number;
+	likes: number;
+	favorites: number;
+	shares: number;
+	danmakus: number;
+	aid: number;
+	replies: number;
+}
+
 /*
  * Fetch video stats from bilibili API and insert into database
  * @returns {Promise<number|VideoSnapshot>}
@@ -17,7 +28,7 @@ export async function insertVideoSnapshot(
 	client: Client,
 	aid: number,
 	task: string,
-): Promise<number | LatestSnapshotType> {
+): Promise<number | SnapshotNumber> {
 	const data = await getVideoInfo(aid, task);
 	if (typeof data == "number") {
 		return data;
@@ -42,7 +53,7 @@ export async function insertVideoSnapshot(
 
 	logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
 
-	const snapshot: LatestSnapshotType = {
+	return {
 		aid,
 		views,
 		danmakus,
@@ -53,6 +64,4 @@ export async function insertVideoSnapshot(
 		favorites,
 		time,
 	};
-
-	return snapshot;
 }

packages/crawler/net/bulkGetVideoStats.ts

@@ -12,8 +12,7 @@ import logger from "log/logger.ts";
  * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
  */
 export async function bulkGetVideoStats(aids: number[]): Promise<MediaListInfoData | number> {
-	const baseURL = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
-	let url = baseURL;
+	let url = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
 	for (const aid of aids) {
 		url += `${aid}:2,`;
 	}

packages/crawler/src/filterWorker.ts

@@ -18,7 +18,7 @@ Deno.addSignalListener("SIGTERM", async () => {
 	Deno.exit();
 });
 
-Akari.init();
+await Akari.init();
 
 const filterWorker = new Worker(
 	"classifyVideo",