diff --git a/.idea/cvsa.iml b/.idea/cvsa.iml index c155925..443232d 100644 --- a/.idea/cvsa.iml +++ b/.idea/cvsa.iml @@ -14,6 +14,9 @@ + + + diff --git a/.idea/deno.xml b/.idea/deno.xml new file mode 100644 index 0000000..2e4b145 --- /dev/null +++ b/.idea/deno.xml @@ -0,0 +1,6 @@ + + + + + \ No newline at end of file diff --git a/deno.json b/deno.json index a3864b1..9104dd2 100644 --- a/deno.json +++ b/deno.json @@ -16,7 +16,6 @@ "imports": { "@astrojs/node": "npm:@astrojs/node@^9.1.3", "@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8", - "@core/db/": "./packages/core/db/", "date-fns": "npm:date-fns@^4.1.0" } } diff --git a/packages/backend/deno.json b/packages/backend/deno.json index 9d3fe45..76b3f5a 100644 --- a/packages/backend/deno.json +++ b/packages/backend/deno.json @@ -4,7 +4,8 @@ "@rabbit-company/argon2id": "jsr:@rabbit-company/argon2id@^2.1.0", "hono": "jsr:@hono/hono@^4.7.5", "zod": "npm:zod", - "yup": "npm:yup" + "yup": "npm:yup", + "@core/": "../core/" }, "tasks": { "dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts", diff --git a/packages/core/db/videoSnapshot.ts b/packages/core/db/videoSnapshot.ts index 68f07ec..48c6aa7 100644 --- a/packages/core/db/videoSnapshot.ts +++ b/packages/core/db/videoSnapshot.ts @@ -1,5 +1,5 @@ import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { VideoSnapshotType } from "@core/db/schema.d.ts"; +import { VideoSnapshotType } from "./schema.d.ts"; export async function getVideoSnapshots(client: Client, aid: number, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') { const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset; diff --git a/packages/core/deno.json b/packages/core/deno.json index e69de29..6467da9 100644 --- a/packages/core/deno.json +++ b/packages/core/deno.json @@ -0,0 +1,4 @@ +{ + "name": "@cvsa/core", + "exports": "./main.ts" +} diff --git a/packages/core/main.ts b/packages/core/main.ts new file mode 100644 index 0000000..1738350 --- /dev/null +++ b/packages/core/main.ts @@ -0,0 +1 @@ +export const DB_VERSION = 10; \ No newline at end of file diff --git a/packages/crawler/db/allData.ts b/packages/crawler/db/allData.ts index 79cfcbd..f2a575a 100644 --- a/packages/crawler/db/allData.ts +++ b/packages/crawler/db/allData.ts @@ -1,5 +1,5 @@ import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { AllDataType, BiliUserType } from "@core/db/schema.d.ts"; +import { AllDataType, BiliUserType } from "@core/db/schema"; import Akari from "ml/akari.ts"; export async function videoExistsInAllData(client: Client, aid: number) { diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index 0be2ccc..f09628c 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -1,5 +1,6 @@ import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { LatestSnapshotType } from "@core/db/schema.d.ts"; +import { LatestSnapshotType } from "@core/db/schema"; +import { SnapshotNumber } from "mq/task/getVideoStats.ts"; export async function getVideosNearMilestone(client: Client) { const queryResult = await client.queryObject(` @@ -18,7 +19,7 @@ export async function getVideosNearMilestone(client: Client) { }); } -export async function getLatestVideoSnapshot(client: Client, aid: number): Promise { +export async function getLatestVideoSnapshot(client: Client, aid: number): Promise { const queryResult = await client.queryObject( ` SELECT * diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index c6a30e0..d8d21bf 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -1,6 +1,5 @@ import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts"; -import { SnapshotScheduleType } from "@core/db/schema.d.ts"; +import { SnapshotScheduleType } from "@core/db/schema"; import logger from "log/logger.ts"; import { MINUTE } from "$std/datetime/constants.ts"; import { redis } from "db/redis.ts"; @@ -11,8 +10,7 @@ const REDIS_KEY = "cvsa:snapshot_window_counts"; function getCurrentWindowIndex(): number { const now = new Date(); const minutesSinceMidnight = now.getHours() * 60 + now.getMinutes(); - const currentWindow = Math.floor(minutesSinceMidnight / 5); - return currentWindow; + return Math.floor(minutesSinceMidnight / 5); } export async function refreshSnapshotWindowCounts(client: Client, redisClient: Redis) { @@ -162,24 +160,6 @@ export async function getLatestSnapshot(client: Client, aid: number): Promise(query, [startTimeString, endTimeString]); - return res.rows[0].count; -} - /* * Creates a new snapshot schedule record. * @param client The database client. @@ -227,7 +207,7 @@ export async function adjustSnapshotTime( const initialOffset = currentWindow + Math.max(targetOffset, 0); - let timePerIteration = 0; + let timePerIteration: number; const MAX_ITERATIONS = 2880; let iters = 0; const t = performance.now(); diff --git a/packages/crawler/deno.json b/packages/crawler/deno.json index 7c10eae..fdd4fb8 100644 --- a/packages/crawler/deno.json +++ b/packages/crawler/deno.json @@ -37,7 +37,8 @@ "express": "npm:express", "src/": "./src/", "onnxruntime": "npm:onnxruntime-node@1.19.2", - "chalk": "npm:chalk" + "chalk": "npm:chalk", + "@core/db/schema": "../core/db/schema.d.ts" }, "exports": "./main.ts" } diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts index 5b89929..a4eebf6 100644 --- a/packages/crawler/mq/exec/snapshotTick.ts +++ b/packages/crawler/mq/exec/snapshotTick.ts @@ -216,7 +216,7 @@ export const regularSnapshotsWorker = async (_job: Job) => { } catch (e) { logger.error(e as Error, "mq", "fn:regularSnapshotsWorker"); } finally { - lockManager.releaseLock("dispatchRegularSnapshots"); + await lockManager.releaseLock("dispatchRegularSnapshots"); client.release(); } }; diff --git a/packages/crawler/mq/scheduler.ts b/packages/crawler/mq/scheduler.ts index 0e8c036..69206b6 100644 --- a/packages/crawler/mq/scheduler.ts +++ b/packages/crawler/mq/scheduler.ts @@ -217,8 +217,7 @@ class NetScheduler { const providerLimiterId = "provider-" + proxyName + "-" + provider; if (!this.proxyLimiters[proxyLimiterId]) { const providerLimiter = this.providerLimiters[providerLimiterId]; - const providerAvailable = await providerLimiter.getAvailability(); - return providerAvailable; + return await providerLimiter.getAvailability(); } const proxyLimiter = this.proxyLimiters[proxyLimiterId]; const providerLimiter = this.providerLimiters[providerLimiterId]; diff --git a/packages/crawler/mq/task/getVideoDetails.ts b/packages/crawler/mq/task/getVideoDetails.ts index 3d9b90e..fa5dd2f 100644 --- a/packages/crawler/mq/task/getVideoDetails.ts +++ b/packages/crawler/mq/task/getVideoDetails.ts @@ -53,7 +53,7 @@ export async function insertVideoInfo(client: Client, aid: number) { query, [aid, stat.view, stat.danmaku, stat.reply, stat.like, stat.coin, stat.share, stat.favorite], ); - + logger.log(`Inserted video metadata for aid: ${aid}`, "mq"); await ClassifyVideoQueue.add("classifyVideo", { aid }); } diff --git a/packages/crawler/mq/task/getVideoStats.ts b/packages/crawler/mq/task/getVideoStats.ts index e1d5a94..3455d93 100644 --- a/packages/crawler/mq/task/getVideoStats.ts +++ b/packages/crawler/mq/task/getVideoStats.ts @@ -1,8 +1,19 @@ import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { getVideoInfo } from "net/getVideoInfo.ts"; -import { LatestSnapshotType } from "@core/db/schema.d.ts"; import logger from "log/logger.ts"; +export interface SnapshotNumber { + time: number; + views: number; + coins: number; + likes: number; + favorites: number; + shares: number; + danmakus: number; + aid: number; + replies: number; +} + /* * Fetch video stats from bilibili API and insert into database * @returns {Promise} @@ -17,7 +28,7 @@ export async function insertVideoSnapshot( client: Client, aid: number, task: string, -): Promise { +): Promise { const data = await getVideoInfo(aid, task); if (typeof data == "number") { return data; @@ -42,7 +53,7 @@ export async function insertVideoSnapshot( logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot"); - const snapshot: LatestSnapshotType = { + return { aid, views, danmakus, @@ -53,6 +64,4 @@ export async function insertVideoSnapshot( favorites, time, }; - - return snapshot; } diff --git a/packages/crawler/net/bulkGetVideoStats.ts b/packages/crawler/net/bulkGetVideoStats.ts index 3ed518c..35c746e 100644 --- a/packages/crawler/net/bulkGetVideoStats.ts +++ b/packages/crawler/net/bulkGetVideoStats.ts @@ -12,8 +12,7 @@ import logger from "log/logger.ts"; * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR` */ export async function bulkGetVideoStats(aids: number[]): Promise { - const baseURL = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`; - let url = baseURL; + let url = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`; for (const aid of aids) { url += `${aid}:2,`; } diff --git a/packages/crawler/src/filterWorker.ts b/packages/crawler/src/filterWorker.ts index cb336c4..54181c4 100644 --- a/packages/crawler/src/filterWorker.ts +++ b/packages/crawler/src/filterWorker.ts @@ -18,7 +18,7 @@ Deno.addSignalListener("SIGTERM", async () => { Deno.exit(); }); -Akari.init(); +await Akari.init(); const filterWorker = new Worker( "classifyVideo",