alikia2x/cvsa
1
0
Fork 0
Code

add: merge turborepo config into main branch

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-12-21 22:33:44 +08:00
commit 1130dcb85f
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG Key ID: 56209E0CCD8420C6
213 changed files with 4351 additions and 4162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.idea/biome.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="BiomeSettings">
<option name="enableLspFormat" value="true" />
<option name="sortImportOnSave" value="true" />
</component>
</project>

2
.idea/data_source_mapping.xml
View File

@ -6,7 +6,5 @@
<file url="file://$PROJECT_DIR$/queries/schedule_count.sql" value="0d2dd3d3-bd27-4e5f-b0fa-ff14fb2a6bef" />
<file url="file://$PROJECT_DIR$/queries/schedule_window.sql" value="0d2dd3d3-bd27-4e5f-b0fa-ff14fb2a6bef" />
<file url="file://$PROJECT_DIR$/queries/snapshots_count.sql" value="0d2dd3d3-bd27-4e5f-b0fa-ff14fb2a6bef" />
<file url="file://$APPLICATION_CONFIG_DIR$/consoles/db/3909a3cf-ec53-4749-8a31-9f90fec87ee1/console.sql" value="3909a3cf-ec53-4749-8a31-9f90fec87ee1" />
<file url="file://$APPLICATION_CONFIG_DIR$/scratches/Query.sql" value="0d2dd3d3-bd27-4e5f-b0fa-ff14fb2a6bef" />
</component>
</project>

1
.idea/prettier.xml
View File

@ -2,5 +2,6 @@
<project version="4">
<component name="PrettierConfiguration">
<option name="myConfigurationMode" value="AUTOMATIC" />
<option name="codeStyleSettingsModifierEnabled" value="false" />
</component>
</project>

8
.prettierrc
View File

@ -1,8 +0,0 @@
{
"useTabs": true,
"tabWidth": 4,
"trailingComma": "none",
"singleQuote": false,
"printWidth": 100,
"endOfLine": "lf"
}

23
biome.json Normal file
View File

@ -0,0 +1,23 @@
{
"$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
"vcs": {
"enabled": true,
"useIgnoreFile": true,
"clientKind": "git"
},
"formatter": {
"enabled": true,
"indentStyle": "tab",
"indentWidth": 4,
"lineWidth": 100
},
"javascript": {
"formatter": {
"enabled": true,
"indentStyle": "tab",
"quoteStyle": "double",
"semicolons": "always",
"trailingCommas": "es5"
}
}
}

34
bun.lock
View File

@ -11,8 +11,10 @@
"postgres": "^3.4.7",
},
"devDependencies": {
"@biomejs/biome": "2.3.8",
"@types/bun": "^1.3.1",
"prettier": "^3.6.2",
"turbo": "^2.6.3",
"vite-tsconfig-paths": "^5.1.4",
"vitest": "^3.2.4",
"vitest-tsconfig-paths": "^3.4.1",
@ -322,6 +324,24 @@
"@babel/types": ["@babel/types@7.28.5", "", { "dependencies": { "@babel/helper-string-parser": "^7.27.1", "@babel/helper-validator-identifier": "^7.28.5" } }, "sha512-qQ5m48eI/MFLQ5PxQj4PFaprjyCTLI37ElWMmNs0K8Lk3dVeOdNpB3ks8jc7yM5CDmVC73eMVk/trk3fgmrUpA=="],
"@biomejs/biome": ["@biomejs/biome@2.3.8", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.8", "@biomejs/cli-darwin-x64": "2.3.8", "@biomejs/cli-linux-arm64": "2.3.8", "@biomejs/cli-linux-arm64-musl": "2.3.8", "@biomejs/cli-linux-x64": "2.3.8", "@biomejs/cli-linux-x64-musl": "2.3.8", "@biomejs/cli-win32-arm64": "2.3.8", "@biomejs/cli-win32-x64": "2.3.8" }, "bin": { "biome": "bin/biome" } }, "sha512-Qjsgoe6FEBxWAUzwFGFrB+1+M8y/y5kwmg5CHac+GSVOdmOIqsAiXM5QMVGZJ1eCUCLlPZtq4aFAQ0eawEUuUA=="],
"@biomejs/cli-darwin-arm64": ["@biomejs/cli-darwin-arm64@2.3.8", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HM4Zg9CGQ3txTPflxD19n8MFPrmUAjaC7PQdLkugeeC0cQ+PiVrd7i09gaBS/11QKsTDBJhVg85CEIK9f50Qww=="],
"@biomejs/cli-darwin-x64": ["@biomejs/cli-darwin-x64@2.3.8", "", { "os": "darwin", "cpu": "x64" }, "sha512-lUDQ03D7y/qEao7RgdjWVGCu+BLYadhKTm40HkpJIi6kn8LSv5PAwRlew/DmwP4YZ9ke9XXoTIQDO1vAnbRZlA=="],
"@biomejs/cli-linux-arm64": ["@biomejs/cli-linux-arm64@2.3.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-Uo1OJnIkJgSgF+USx970fsM/drtPcQ39I+JO+Fjsaa9ZdCN1oysQmy6oAGbyESlouz+rzEckLTF6DS7cWse95g=="],
"@biomejs/cli-linux-arm64-musl": ["@biomejs/cli-linux-arm64-musl@2.3.8", "", { "os": "linux", "cpu": "arm64" }, "sha512-PShR4mM0sjksUMyxbyPNMxoKFPVF48fU8Qe8Sfx6w6F42verbwRLbz+QiKNiDPRJwUoMG1nPM50OBL3aOnTevA=="],
"@biomejs/cli-linux-x64": ["@biomejs/cli-linux-x64@2.3.8", "", { "os": "linux", "cpu": "x64" }, "sha512-QDPMD5bQz6qOVb3kiBui0zKZXASLo0NIQ9JVJio5RveBEFgDgsvJFUvZIbMbUZT3T00M/1wdzwWXk4GIh0KaAw=="],
"@biomejs/cli-linux-x64-musl": ["@biomejs/cli-linux-x64-musl@2.3.8", "", { "os": "linux", "cpu": "x64" }, "sha512-YGLkqU91r1276uwSjiUD/xaVikdxgV1QpsicT0bIA1TaieM6E5ibMZeSyjQ/izBn4tKQthUSsVZacmoJfa3pDA=="],
"@biomejs/cli-win32-arm64": ["@biomejs/cli-win32-arm64@2.3.8", "", { "os": "win32", "cpu": "arm64" }, "sha512-H4IoCHvL1fXKDrTALeTKMiE7GGWFAraDwBYFquE/L/5r1927Te0mYIGseXi4F+lrrwhSWbSGt5qPFswNoBaCxg=="],
"@biomejs/cli-win32-x64": ["@biomejs/cli-win32-x64@2.3.8", "", { "os": "win32", "cpu": "x64" }, "sha512-RguzimPoZWtBapfKhKjcWXBVI91tiSprqdBYu7tWhgN8pKRZhw24rFeNZTNf6UiBfjCYCi9eFQs/JzJZIhuK4w=="],
"@borewit/text-codec": ["@borewit/text-codec@0.1.1", "", {}, "sha512-5L/uBxmjaCIX5h8Z+uu+kA9BQLkc/Wl06UGR5ajNRxu+/XjonB5i8JpgFMrPj3LXTCPA0pv8yxUvbUi+QthGGA=="],
"@bull-board/api": ["@bull-board/api@6.15.0", "", { "dependencies": { "redis-info": "^3.1.0" }, "peerDependencies": { "@bull-board/ui": "6.15.0" } }, "sha512-z8qLZ4uv83hZNu+0YnHzhVoWv1grULuYh80FdC2xXLg8M1EwsOZD9cJ5CNpgBFqHb+NVByTmf5FltIvXdOU8tQ=="],
@ -2116,6 +2136,20 @@
"tunnel-agent": ["tunnel-agent@0.6.0", "", { "dependencies": { "safe-buffer": "^5.0.1" } }, "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w=="],
"turbo": ["turbo@2.6.3", "", { "optionalDependencies": { "turbo-darwin-64": "2.6.3", "turbo-darwin-arm64": "2.6.3", "turbo-linux-64": "2.6.3", "turbo-linux-arm64": "2.6.3", "turbo-windows-64": "2.6.3", "turbo-windows-arm64": "2.6.3" }, "bin": { "turbo": "bin/turbo" } }, "sha512-bf6YKUv11l5Xfcmg76PyWoy/e2vbkkxFNBGJSnfdSXQC33ZiUfutYh6IXidc5MhsnrFkWfdNNLyaRk+kHMLlwA=="],
"turbo-darwin-64": ["turbo-darwin-64@2.6.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-BlJJDc1CQ7SK5Y5qnl7AzpkvKSnpkfPmnA+HeU/sgny3oHZckPV2776ebO2M33CYDSor7+8HQwaodY++IINhYg=="],
"turbo-darwin-arm64": ["turbo-darwin-arm64@2.6.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-MwVt7rBKiOK7zdYerenfCRTypefw4kZCue35IJga9CH1+S50+KTiCkT6LBqo0hHeoH2iKuI0ldTF2a0aB72z3w=="],
"turbo-linux-64": ["turbo-linux-64@2.6.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cqpcw+dXxbnPtNnzeeSyWprjmuFVpHJqKcs7Jym5oXlu/ZcovEASUIUZVN3OGEM6Y/OTyyw0z09tOHNt5yBAVg=="],
"turbo-linux-arm64": ["turbo-linux-arm64@2.6.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-MterpZQmjXyr4uM7zOgFSFL3oRdNKeflY7nsjxJb2TklsYqiu3Z9pQ4zRVFFH8n0mLGna7MbQMZuKoWqqHb45w=="],
"turbo-windows-64": ["turbo-windows-64@2.6.3", "", { "os": "win32", "cpu": "x64" }, "sha512-biDU70v9dLwnBdLf+daoDlNJVvqOOP8YEjqNipBHzgclbQlXbsi6Gqqelp5er81Qo3BiRgmTNx79oaZQTPb07Q=="],
"turbo-windows-arm64": ["turbo-windows-arm64@2.6.3", "", { "os": "win32", "cpu": "arm64" }, "sha512-dDHVKpSeukah3VsI/xMEKeTnV9V9cjlpFSUs4bmsUiLu3Yv2ENlgVEZv65wxbeE0bh0jjpmElDT+P1KaCxArQQ=="],
"tw-animate-css": ["tw-animate-css@1.4.0", "", {}, "sha512-7bziOlRqH0hJx80h/3mbicLW7o8qLsH5+RaLR2t+OHM3D0JlWGODQKQ4cxbK7WlvmUxpcj6Kgu6EKqjrGFe3QQ=="],
"type-check": ["type-check@0.4.0", "", { "dependencies": { "prelude-ls": "^1.2.1" } }, "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew=="],

102
ecosystem.config.mjs
View File

@ -1,53 +1,53 @@
import 'dotenv/config'
import "dotenv/config";
export const apps = [
{
name: 'crawler-jobadder',
script: 'src/jobAdder.wrapper.ts',
cwd: './packages/crawler',
interpreter: 'bun',
},
{
name: 'crawler-worker',
script: 'src/worker.ts',
cwd: './packages/crawler',
interpreter: 'bun',
env: {
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log"
}
},
{
name: 'crawler-filter',
script: 'src/filterWorker.wrapper.ts',
cwd: './packages/crawler',
interpreter: 'bun',
env: {
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log"
}
},
{
name: 'ml-api',
script: 'start.py',
cwd: './ml/api',
interpreter: process.env.PYTHON_INTERPRETER || 'python3',
env: {
PYTHONPATH: './ml/api:./ml/filter',
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log"
}
},
{
name: 'cvsa-be',
script: 'src/index.ts',
cwd: './packages/backend',
interpreter: 'bun',
env: {
NODE_ENV: 'production'
}
},
]
{
name: "crawler-jobadder",
script: "src/jobAdder.wrapper.ts",
cwd: "./packages/crawler",
interpreter: "bun",
},
{
name: "crawler-worker",
script: "src/worker.ts",
cwd: "./packages/crawler",
interpreter: "bun",
env: {
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log",
},
},
{
name: "crawler-filter",
script: "src/filterWorker.wrapper.ts",
cwd: "./packages/crawler",
interpreter: "bun",
env: {
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log",
},
},
{
name: "ml-api",
script: "start.py",
cwd: "./ml/api",
interpreter: process.env.PYTHON_INTERPRETER || "python3",
env: {
PYTHONPATH: "./ml/api:./ml/filter",
LOG_VERBOSE: "logs/verbose.log",
LOG_WARN: "logs/warn.log",
LOG_ERR: "logs/error.log",
},
},
{
name: "cvsa-be",
script: "src/index.ts",
cwd: "./packages/backend",
interpreter: "bun",
env: {
NODE_ENV: "production",
},
},
];

10
package.json
View File

@ -3,6 +3,11 @@
"version": "5.5.0",
"private": false,
"type": "module",
"scripts": {
"biome:format": "bunx --bun biome format --write",
"biome:lint": "bunx --bun biome lint",
"biome:check": "bunx --bun biome check --write"
},
"workspaces": {
"packages": [
"packages/core",
@ -23,10 +28,13 @@
"postgres": "^3.4.7"
},
"devDependencies": {
"@biomejs/biome": "2.3.8",
"@types/bun": "^1.3.1",
"prettier": "^3.6.2",
"turbo": "^2.6.3",
"vite-tsconfig-paths": "^5.1.4",
"vitest": "^3.2.4",
"vitest-tsconfig-paths": "^3.4.1"
}
},
"packageManager": "bun@1.3.3"
}

7
packages/backend/biome.json Normal file
View File

@ -0,0 +1,7 @@
{
"extends": "//",
"$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
"linter": {
"enabled": true
}
}

20
packages/backend/lib/auth.ts
View File

@ -1,14 +1,14 @@
import Argon2id from "@rabbit-company/argon2id";
import { generate as generateId } from "@alikia/random-key";
import {
db,
usersInCredentials,
loginSessionsInCredentials,
UserType,
SessionType
type SessionType,
type UserType,
usersInCredentials,
} from "@core/drizzle";
import { eq, and, isNull } from "drizzle-orm";
import { generate as generateId } from "@alikia/random-key";
import logger from "@core/log";
import Argon2id from "@rabbit-company/argon2id";
import { and, eq, isNull } from "drizzle-orm";
export async function verifyUser(
username: string,
@ -36,7 +36,7 @@ export async function verifyUser(
nickname: foundUser.nickname,
role: foundUser.role,
unqId: foundUser.unqId,
createdAt: foundUser.createdAt
createdAt: foundUser.createdAt,
};
}
@ -57,7 +57,7 @@ export async function createSession(
ipAddress,
userAgent,
lastUsedAt: new Date().toISOString(),
expireAt: expireAt.toISOString()
expireAt: expireAt.toISOString(),
});
} catch (error) {
logger.error(error as Error);
@ -108,7 +108,7 @@ export async function validateSession(
return {
user: users[0],
session: session
session: session,
};
}
@ -116,7 +116,7 @@ export async function deactivateSession(sessionId: string): Promise<boolean> {
const result = await db
.update(loginSessionsInCredentials)
.set({
deactivatedAt: new Date().toISOString()
deactivatedAt: new Date().toISOString(),
})
.where(eq(loginSessionsInCredentials.id, sessionId));

4
packages/backend/lib/bilibiliID.ts
View File

@ -39,12 +39,12 @@ export function detectBiliID(id: string) {
if (bvSchema.safeParse(id).success) {
return {
type: "bv" as const,
id: id as `BV1${string}`
id: id as `BV1${string}`,
};
} else if (avSchema.safeParse(id).success) {
return {
type: "av" as const,
id: id as `av${string}`
id: id as `av${string}`,
};
}
return null;

6
packages/backend/lib/mq.ts
View File

@ -1,10 +1,10 @@
import { Queue, ConnectionOptions } from "bullmq";
import { redis } from "@core/db/redis";
import { type ConnectionOptions, Queue } from "bullmq";
export const LatestVideosQueue = new Queue("latestVideos", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
export const SnapshotQueue = new Queue("snapshot", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});

10
packages/backend/lib/schema.ts
View File

@ -10,7 +10,7 @@ const videoStatsSchema = z.object({
share: z.number(),
now_rank: z.number(),
his_rank: z.number(),
like: z.number()
like: z.number(),
});
export const BiliAPIVideoMetadataSchema = z.object({
@ -32,9 +32,9 @@ export const BiliAPIVideoMetadataSchema = z.object({
owner: z.object({
mid: z.number(),
name: z.string(),
face: z.string()
face: z.string(),
}),
stat: videoStatsSchema
stat: videoStatsSchema,
});
export const BiliVideoSchema = z.object({
@ -49,7 +49,7 @@ export const BiliVideoSchema = z.object({
tags: z.string().nullable(),
title: z.string().nullable(),
status: z.number(),
coverUrl: z.string().nullable()
coverUrl: z.string().nullable(),
});
export type BiliVideoType = z.infer<typeof BiliVideoSchema>;
@ -66,5 +66,5 @@ export const SongSchema = z.object({
updatedAt: z.string(),
deleted: z.boolean(),
image: z.string().nullable(),
producer: z.string().nullable()
producer: z.string().nullable(),
});

42
packages/backend/lib/singers.ts
View File

@ -9,99 +9,99 @@ export const singers: Singer[] = [
{
name: "洛天依",
color: "#66CCFF",
birthday: "0712"
birthday: "0712",
},
{
name: "言和",
color: "#00FFCC",
birthday: "0711"
birthday: "0711",
},
{
name: "乐正绫",
color: "#EE0000",
birthday: "0412"
birthday: "0412",
},
{
name: "乐正龙牙",
color: "#006666",
birthday: "1002"
birthday: "1002",
},
{
name: "徵羽摩柯",
color: "#0080FF",
birthday: "1210"
birthday: "1210",
},
{
name: "墨清弦",
color: "#FFFF00",
birthday: "0520"
birthday: "0520",
},
{
name: "星尘",
color: "#9999FF",
birthday: "0812"
birthday: "0812",
},
{
name: "永夜Minus",
color: "#613c8a",
birthday: "1208"
birthday: "1208",
},
{
name: "心华",
color: "#EE82EE",
birthday: "0210"
birthday: "0210",
},
{
name: "海伊",
color: "#3399FF",
birthday: "0722"
birthday: "0722",
},
{
name: "苍穹",
color: "#8BC0B5",
birthday: "0520"
birthday: "0520",
},
{
name: "赤羽",
color: "#FF4004",
birthday: "1126"
birthday: "1126",
},
{
name: "诗岸",
color: "#F6BE72",
birthday: "0119"
birthday: "0119",
},
{
name: "牧心",
color: "#2A2859",
birthday: "0807"
birthday: "0807",
},
{
name: "起礼",
color: "#FF0099",
birthday: "0713"
birthday: "0713",
},
{
name: "起复",
color: "#99FF00",
birthday: "0713"
birthday: "0713",
},
{
name: "夏语遥",
color: "#34CCCC",
birthday: "1110"
}
birthday: "1110",
},
];
export const specialSingers = [
{
name: "雅音宫羽",
message: "你是我最真模样，从来不曾遗忘。"
message: "你是我最真模样，从来不曾遗忘。",
},
{
name: "初音未来",
message: "初始之音，响彻未来!"
}
message: "初始之音，响彻未来!",
},
];
export const pickSinger = () => {

14
packages/backend/middlewares/auth.ts
View File

@ -1,6 +1,6 @@
import { Elysia } from "elysia";
import { validateSession } from "@backend/lib/auth";
import { SessionType, UserType } from "@core/drizzle";
import type { SessionType, UserType } from "@core/drizzle";
import { Elysia } from "elysia";
export interface AuthenticatedContext {
user: UserType;
@ -42,7 +42,7 @@ export const requireAuth = new Elysia({ name: "require-auth" })
return {
user: null,
session: null,
isAuthenticated: false
isAuthenticated: false,
};
}
@ -54,7 +54,7 @@ export const requireAuth = new Elysia({ name: "require-auth" })
return {
user: null,
session: null,
isAuthenticated: false
isAuthenticated: false,
};
}
@ -62,13 +62,13 @@ export const requireAuth = new Elysia({ name: "require-auth" })
return {
user: validationResult.user,
session: validationResult.session,
isAuthenticated: true
isAuthenticated: true,
};
})
.onBeforeHandle({ as: "scoped" }, ({ user, session, status }) => {
.onBeforeHandle({ as: "scoped" }, ({ user, status }) => {
if (!user) {
return status(401, {
message: "Authentication required."
message: "Authentication required.",
});
}
})

10
packages/backend/middlewares/captcha.ts
View File

@ -1,17 +1,17 @@
import { Elysia } from "elysia";
import { jwt } from "@elysiajs/jwt";
import { redis } from "@core/db/redis";
import { jwt } from "@elysiajs/jwt";
import { Elysia } from "elysia";
interface JWTPayload {
id: string;
[key: string]: any;
[key: string]: unknown;
}
export const captchaMiddleware = new Elysia({ name: "captcha" })
.use(
jwt({
name: "captchaJwt",
secret: process.env.JWT_SECRET || "default-secret-key"
secret: process.env.JWT_SECRET || "default-secret-key",
})
)
.derive(async ({ request, captchaJwt, set }) => {
@ -44,7 +44,7 @@ export const captchaMiddleware = new Elysia({ name: "captcha" })
return {
captchaVerified: true,
userId: payload.id
userId: payload.id,
};
} catch (error) {
if (error instanceof Error) {

15
packages/backend/middlewares/openapi.ts
View File

@ -1,22 +1,21 @@
import openapi from "@elysiajs/openapi";
import pkg from "../package.json";
import openapi, { fromTypes } from "@elysiajs/openapi";
import * as z from "zod";
import { fromTypes } from "@elysiajs/openapi";
import pkg from "../package.json";
export const openAPIMiddleware = openapi({
documentation: {
info: {
title: "CVSA API Docs",
version: pkg.version
}
version: pkg.version,
},
},
mapJsonSchema: {
zod: z.toJSONSchema
zod: z.toJSONSchema,
},
references: fromTypes(),
scalar: {
theme: "kepler",
hideClientButton: true,
hideDarkModeToggle: true
}
hideDarkModeToggle: true,
},
});

187
packages/backend/middlewares/timing.ts
View File

@ -1,35 +1,20 @@
import { Elysia, type MapResponse, type Context, type TraceEvent, type TraceProcess } from "elysia";
import { type Context, Elysia, type MapResponse, type TraceEvent, type TraceProcess } from "elysia";
type MaybePromise<T> = T | Promise<T>;
class TimeLogger {
private startTimes: Map<string, number>;
private durations: Map<string, number>;
private totalStartTime: number | null;
constructor() {
this.startTimes = new Map();
this.durations = new Map();
this.totalStartTime = null;
}
startTime(name: string) {
this.startTimes.set(name, performance.now());
}
endTime(name: string) {
const startTime = this.startTimes.get(name);
if (startTime !== undefined) {
const duration = performance.now() - startTime;
this.durations.set(name, duration);
this.startTimes.delete(name);
}
}
getCompletedDurations() {
return Array.from(this.durations.entries()).map(([name, duration]) => ({
name,
duration
duration,
}));
}
@ -108,7 +93,7 @@ export interface ServerTimingOptions {
total?: boolean;
};
/**
* Determine whether or not Server Timing should be enabled
* Determine whether Server Timing should be enabled
*
* @default NODE_ENV !== 'production'
*/
@ -129,7 +114,7 @@ export interface ServerTimingOptions {
const getLabel = (
event: TraceEvent,
listener: (callback: (process: TraceProcess<"begin", true>) => unknown) => unknown,
listener: (callback: (process: TraceProcess<"begin">) => unknown) => unknown,
write: (value: string) => void
) => {
listener(async ({ onStop, onEvent, total }) => {
@ -137,13 +122,13 @@ const getLabel = (
if (total === 0) return;
onEvent(({ name, index, onStop }) => {
await onEvent(({ name, index, onStop }) => {
onStop(({ elapsed }) => {
label += `${event}.${index}.${name || "anon"};dur=${elapsed},`;
});
});
onStop(({ elapsed }) => {
await onStop(({ elapsed }) => {
label += `${event};dur=${elapsed},`;
write(label);
@ -163,99 +148,91 @@ export const serverTiming = ({
afterHandle: traceAfterHandle = true,
error: traceError = true,
mapResponse: traceMapResponse = true,
total: traceTotal = true
total: traceTotal = true,
} = {},
mapResponse
}: ServerTimingOptions = {}) => {
const app = new Elysia().decorate("timeLog", new TimeLogger()).trace(
{ as: "global" },
async ({
onRequest,
onParse,
onTransform,
onBeforeHandle,
onHandle,
onAfterHandle,
onMapResponse,
onError,
set,
context,
response,
context: {
request: { method }
}
}) => {
if (!enabled) return;
let label = "";
return new Elysia()
.decorate("timeLog", new TimeLogger())
.trace(
{ as: "global" },
async ({
onRequest,
onParse,
onTransform,
onBeforeHandle,
onHandle,
onAfterHandle,
onMapResponse,
onError,
set,
context,
}) => {
if (!enabled) return;
let label = "";
const write = (nextValue: string) => {
label += nextValue;
};
const write = (nextValue: string) => {
label += nextValue;
};
let start: number;
await onRequest(() => {
context.timeLog.startTotal();
});
onRequest(({ begin }) => {
context.timeLog.startTotal();
start = begin;
});
if (traceRequest) getLabel("request", onRequest, write);
if (traceParse) getLabel("parse", onParse, write);
if (traceTransform) getLabel("transform", onTransform, write);
if (traceBeforeHandle) getLabel("beforeHandle", onBeforeHandle, write);
if (traceAfterHandle) getLabel("afterHandle", onAfterHandle, write);
if (traceError) getLabel("error", onError, write);
if (traceMapResponse) getLabel("mapResponse", onMapResponse, write);
if (traceRequest) getLabel("request", onRequest, write);
if (traceParse) getLabel("parse", onParse, write);
if (traceTransform) getLabel("transform", onTransform, write);
if (traceBeforeHandle) getLabel("beforeHandle", onBeforeHandle, write);
if (traceAfterHandle) getLabel("afterHandle", onAfterHandle, write);
if (traceError) getLabel("error", onError, write);
if (traceMapResponse) getLabel("mapResponse", onMapResponse, write);
if (traceHandle)
await onHandle(({ name, onStop }) => {
onStop(({ elapsed }) => {
label += `handle.${name};dur=${elapsed},`;
});
});
if (traceHandle)
onHandle(({ name, onStop }) => {
onStop(({ elapsed }) => {
label += `handle.${name};dur=${elapsed},`;
await onMapResponse(({ onStop }) => {
onStop(async () => {
const completedDurations = context.timeLog.getCompletedDurations();
if (completedDurations.length > 0) {
label += `${completedDurations
.map(({ name, duration }) => `${name};dur=${duration}`)
.join(", ")},`;
}
const elapsed = context.timeLog.endTotal();
let allowed = allow;
if (allowed instanceof Promise) allowed = await allowed;
if (traceTotal) label += `total;dur=${elapsed}`;
else label = label.slice(0, -1);
// ? Must wait until request is reported
switch (typeof allowed) {
case "boolean":
if (!allowed) delete set.headers["Server-Timing"];
set.headers["Server-Timing"] = label;
break;
case "function":
if ((await allowed(context)) === false)
delete set.headers["Server-Timing"];
set.headers["Server-Timing"] = label;
break;
default:
set.headers["Server-Timing"] = label;
}
});
});
onMapResponse(({ onStop }) => {
onStop(async ({ end }) => {
const completedDurations = context.timeLog.getCompletedDurations();
if (completedDurations.length > 0) {
label +=
completedDurations
.map(({ name, duration }) => `${name};dur=${duration}`)
.join(", ") + ",";
}
const elapsed = context.timeLog.endTotal();
let allowed = allow;
if (allowed instanceof Promise) allowed = await allowed;
if (traceTotal) label += `total;dur=${elapsed}`;
else label = label.slice(0, -1);
// ? Must wait until request is reported
switch (typeof allowed) {
case "boolean":
if (allowed === false) delete set.headers["Server-Timing"];
set.headers["Server-Timing"] = label;
break;
case "function":
if ((await allowed(context)) === false)
delete set.headers["Server-Timing"];
set.headers["Server-Timing"] = label;
break;
default:
set.headers["Server-Timing"] = label;
}
});
});
}
);
return app;
}
);
};
export default serverTiming;

3
packages/backend/package.json
View File

@ -3,8 +3,7 @@
"version": "1.1.0",
"scripts": {
"dev": "NODE_ENV=development bun run --watch src/index.ts",
"start": "NODE_ENV=production bun run src/index.ts",
"format": "prettier --write ."
"start": "NODE_ENV=production bun run src/index.ts"
},
"dependencies": {
"@alikia/random-key": "^2.0.0",

20
packages/backend/routes/auth/login.ts
View File

@ -1,6 +1,6 @@
import { createSession, getSessionExpirationDate, verifyUser } from "@backend/lib/auth";
import { Elysia, t } from "elysia";
import { ip } from "elysia-ip";
import { verifyUser, createSession, getSessionExpirationDate } from "@backend/lib/auth";
export const loginHandler = new Elysia({ prefix: "/auth" }).use(ip()).post(
"/session",
@ -29,9 +29,9 @@ export const loginHandler = new Elysia({ prefix: "/auth" }).use(ip()).post(
id: user.id,
username: user.username,
nickname: user.nickname,
role: user.role
role: user.role,
},
sessionID: sessionId
sessionID: sessionId,
};
},
{
@ -42,24 +42,24 @@ export const loginHandler = new Elysia({ prefix: "/auth" }).use(ip()).post(
id: t.Integer(),
username: t.String(),
nickname: t.Optional(t.String()),
role: t.String()
role: t.String(),
}),
sessionID: t.String()
sessionID: t.String(),
}),
401: t.Object({
message: t.String()
})
message: t.String(),
}),
},
body: t.Object({
username: t.String(),
password: t.String()
password: t.String(),
}),
detail: {
summary: "User login",
description:
"This endpoint authenticates users by verifying their credentials and creates a new session. \
Upon successful authentication, it returns user information and sets a secure HTTP-only cookie \
for session management. The session includes IP address and user agent tracking for security purposes."
}
for session management. The session includes IP address and user agent tracking for security purposes.",
},
}
);

12
packages/backend/routes/auth/logout.ts
View File

@ -1,6 +1,6 @@
import { Elysia, t } from "elysia";
import { deactivateSession } from "@backend/lib/auth";
import requireAuth from "@backend/middlewares/auth";
import { Elysia, t } from "elysia";
export const logoutHandler = new Elysia({ prefix: "/auth" }).use(requireAuth).delete(
"/session",
@ -20,18 +20,18 @@ export const logoutHandler = new Elysia({ prefix: "/auth" }).use(requireAuth).de
{
response: {
200: t.Object({
message: t.String()
message: t.String(),
}),
401: t.Object({
message: t.String()
})
message: t.String(),
}),
},
detail: {
summary: "Logout current session",
description:
"This endpoint logs out the current user by deactivating their session and removing the session cookie. \
It requires an active session cookie to be present in the request. After successful logout, the session \
is invalidated and cannot be used again."
}
is invalidated and cannot be used again.",
},
}
);

12
packages/backend/routes/auth/user.ts
View File

@ -1,5 +1,5 @@
import { Elysia, t } from "elysia";
import requireAuth from "@backend/middlewares/auth";
import { Elysia, t } from "elysia";
export const getCurrentUserHandler = new Elysia().use(requireAuth).get(
"/user",
@ -11,7 +11,7 @@ export const getCurrentUserHandler = new Elysia().use(requireAuth).get(
id: user.id,
username: user.username,
nickname: user.nickname,
role: user.role
role: user.role,
};
},
{
@ -20,11 +20,11 @@ export const getCurrentUserHandler = new Elysia().use(requireAuth).get(
id: t.Integer(),
username: t.String(),
nickname: t.Union([t.String(), t.Null()]),
role: t.String()
role: t.String(),
}),
401: t.Object({
message: t.String()
})
}
message: t.String(),
}),
},
}
);

18
packages/backend/routes/ping/index.ts
View File

@ -12,13 +12,13 @@ export const pingHandler = new Elysia({ prefix: "/ping" }).use(ip()).get(
ip: ip,
method: request.method,
body: body,
url: request.url
url: request.url,
},
response: {
time: Date.now(),
status: 200,
version: VERSION
}
version: VERSION,
},
};
},
{
@ -30,20 +30,20 @@ export const pingHandler = new Elysia({ prefix: "/ping" }).use(ip()).get(
ip: t.Optional(t.String()),
method: t.String(),
body: t.Optional(t.Union([t.String(), t.Null()])),
url: t.String()
url: t.String(),
}),
response: t.Object({
time: t.Number(),
status: t.Number(),
version: t.String()
})
})
version: t.String(),
}),
}),
},
body: t.Optional(t.String()),
detail: {
summary: "Send a ping",
description:
"This endpoint returns a 'pong' message along with comprehensive information about the incoming request and the server's current status, including request headers, IP address, and server version. It's useful for monitoring API availability and debugging."
}
"This endpoint returns a 'pong' message along with comprehensive information about the incoming request and the server's current status, including request headers, IP address, and server version. It's useful for monitoring API availability and debugging.",
},
}
);

23
packages/backend/routes/root/index.ts
View File

@ -1,4 +1,9 @@
import { getSingerForBirthday, pickSinger, pickSpecialSinger, Singer } from "@backend/lib/singers";
import {
getSingerForBirthday,
pickSinger,
pickSpecialSinger,
type Singer,
} from "@backend/lib/singers";
import { VERSION } from "@backend/src";
import { Elysia, t } from "elysia";
@ -6,7 +11,7 @@ const SingerObj = t.Object({
name: t.String(),
color: t.Optional(t.String()),
birthday: t.Optional(t.String()),
message: t.Optional(t.String())
message: t.Optional(t.String()),
});
export const rootHandler = new Elysia().get(
@ -29,12 +34,12 @@ export const rootHandler = new Elysia().get(
project: {
name: "中V档案馆",
mascot: "知夏",
quote: "星河知海夏生光"
quote: "星河知海夏生光",
},
status: 200,
version: VERSION,
time: Date.now(),
singer: singer
singer: singer,
};
},
{
@ -43,19 +48,19 @@ export const rootHandler = new Elysia().get(
project: t.Object({
name: t.String(),
mascot: t.String(),
quote: t.String()
quote: t.String(),
}),
status: t.Number(),
version: t.String(),
time: t.Number(),
singer: t.Union([SingerObj, t.Array(SingerObj)])
})
singer: t.Union([SingerObj, t.Array(SingerObj)]),
}),
},
detail: {
summary: "Root route",
description:
"The root path. It returns a JSON object containing a random virtual singer, \
backend version, current server time and other miscellaneous information."
}
backend version, current server time and other miscellaneous information.",
},
}
);

59
packages/backend/routes/search/index.ts
View File

@ -1,12 +1,13 @@
import { Elysia } from "elysia";
import { db, bilibiliMetadata, latestVideoSnapshot, songs } from "@core/drizzle";
import { eq, ilike } from "drizzle-orm";
import { BiliAPIVideoMetadataSchema, BiliVideoSchema, SongSchema } from "@backend/lib/schema";
import { z } from "zod";
import { getVideoInfo } from "@core/net/getVideoInfo";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { retrieveVideoInfoFromCache } from "../video/metadata";
import { BiliAPIVideoMetadataSchema, BiliVideoSchema, SongSchema } from "@backend/lib/schema";
import { redis } from "@core/db/redis";
import { bilibiliMetadata, db, latestVideoSnapshot, songs } from "@core/drizzle";
import type { VideoInfoData } from "@core/net/bilibili";
import { getVideoInfo } from "@core/net/getVideoInfo";
import { eq, ilike } from "drizzle-orm";
import { Elysia } from "elysia";
import { z } from "zod";
import { retrieveVideoInfoFromCache } from "../video/metadata";
const getSongSearchResult = async (searchQuery: string) => {
const data = await db
@ -26,7 +27,7 @@ const getSongSearchResult = async (searchQuery: string) => {
data: song,
occurrences,
viewsLog,
lengthRatio
lengthRatio,
};
})
.filter((d) => d !== null);
@ -53,7 +54,7 @@ const getSongSearchResult = async (searchQuery: string) => {
return {
type: result.type,
data: result.data.songs,
rank: Math.min(Math.max(rank, 0), 1) // Ensure rank is between 0 and 1
rank: Math.min(Math.max(rank, 0), 1), // Ensure rank is between 0 and 1
};
});
@ -71,21 +72,21 @@ const getDBVideoSearchResult = async (searchQuery: string) => {
return results.map((video) => ({
type: "bili-video-db" as "bili-video-db",
data: { views: video.latest_video_snapshot.views, ...video.bilibili_metadata },
rank: 1 // Exact match
rank: 1, // Exact match
}));
};
const getVideoSearchResult = async (searchQuery: string) => {
const aid = biliIDToAID(searchQuery);
if (!aid) return [];
let data;
let data: VideoInfoData;
const cachedData = await retrieveVideoInfoFromCache(aid);
if (cachedData) {
data = cachedData;
} else {
data = await getVideoInfo(aid, "getVideoInfo");
if (typeof data === "number") return [];
data = data.data;
const result = await getVideoInfo(aid, "getVideoInfo");
if (typeof result === "number") return [];
data = result.data;
const cacheKey = `cvsa:videoInfo:av${aid}`;
await redis.setex(cacheKey, 60, JSON.stringify(data));
}
@ -93,13 +94,13 @@ const getVideoSearchResult = async (searchQuery: string) => {
{
type: "bili-video" as "bili-video",
data: data,
rank: 0.99 // Exact match
}
rank: 0.99, // Exact match
},
];
};
const BiliVideoDataSchema = BiliVideoSchema.extend({
views: z.number()
views: z.number(),
});
export const searchHandler = new Elysia({ prefix: "/search" }).get(
@ -110,7 +111,7 @@ export const searchHandler = new Elysia({ prefix: "/search" }).get(
const [songResults, videoResults, dbVideoResults] = await Promise.all([
getSongSearchResult(searchQuery),
getVideoSearchResult(searchQuery),
getDBVideoSearchResult(searchQuery)
getDBVideoSearchResult(searchQuery),
]);
const combinedResults = [...songResults, ...videoResults, ...dbVideoResults];
@ -118,7 +119,7 @@ export const searchHandler = new Elysia({ prefix: "/search" }).get(
const end = performance.now();
return {
data,
elapsedMs: end - start
elapsedMs: end - start,
};
},
{
@ -130,27 +131,27 @@ export const searchHandler = new Elysia({ prefix: "/search" }).get(
z.object({
type: z.literal("song"),
data: SongSchema,
rank: z.number()
rank: z.number(),
}),
z.object({
type: z.literal("bili-video-db"),
data: BiliVideoDataSchema,
rank: z.number()
rank: z.number(),
}),
z.object({
type: z.literal("bili-video"),
data: BiliAPIVideoMetadataSchema,
rank: z.number()
})
rank: z.number(),
}),
])
)
),
}),
404: z.object({
message: z.string()
})
message: z.string(),
}),
},
query: z.object({
query: z.string()
query: z.string(),
}),
detail: {
summary: "Search songs and videos",
@ -158,7 +159,7 @@ export const searchHandler = new Elysia({ prefix: "/search" }).get(
"This endpoint performs a comprehensive search across songs and videos in the database. \
It searches for songs by name and videos by bilibili ID (av/BV format). The results are ranked \
by relevance using a weighted algorithm that considers search term frequency, title length, \
and view count. Returns search results with performance timing information."
}
and view count. Returns search results with performance timing information.",
},
}
);

52
packages/backend/routes/song/add.ts
View File

@ -1,9 +1,9 @@
import { Elysia, t } from "elysia";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { requireAuth } from "@backend/middlewares/auth";
import { LatestVideosQueue } from "@backend/lib/mq";
import { requireAuth } from "@backend/middlewares/auth";
import { db, songs } from "@core/drizzle";
import { eq, and } from "drizzle-orm";
import { and, eq } from "drizzle-orm";
import { Elysia, t } from "elysia";
export const addSongHandler = new Elysia()
.use(requireAuth)
@ -15,7 +15,7 @@ export const addSongHandler = new Elysia()
if (!aid) {
return status(400, {
message:
"We cannot parse the video ID, or we currently do not support this format."
"We cannot parse the video ID, or we currently do not support this format.",
});
}
const aidExistsInSongs = await db
@ -26,42 +26,42 @@ export const addSongHandler = new Elysia()
if (aidExistsInSongs.length > 0) {
return {
jobID: -1,
message: "Video already exists in the songs table."
message: "Video already exists in the songs table.",
};
}
const job = await LatestVideosQueue.add("getVideoInfo", {
aid: aid,
insertSongs: true,
uid: user!.unqId
uid: user!.unqId,
});
if (!job.id) {
return status(500, {
message: "Failed to enqueue job to add song."
message: "Failed to enqueue job to add song.",
});
}
return status(201, {
message: "Successfully created import session.",
jobID: job.id
jobID: job.id,
});
},
{
response: {
201: t.Object({
message: t.String(),
jobID: t.String()
jobID: t.String(),
}),
400: t.Object({
message: t.String()
message: t.String(),
}),
401: t.Object({
message: t.String()
message: t.String(),
}),
500: t.Object({
message: t.String()
})
message: t.String(),
}),
},
body: t.Object({
id: t.String()
id: t.String(),
}),
detail: {
summary: "Import song from bilibili",
@ -69,8 +69,8 @@ export const addSongHandler = new Elysia()
"This endpoint allows authenticated users to import a song from bilibili by providing a video ID. \
The video ID can be in av or BV format. The system validates the ID format, checks if the video already \
exists in the database, and if not, creates a background job to fetch video metadata and add it to the songs collection. \
Returns the job ID for tracking the import progress."
}
Returns the job ID for tracking the import progress.",
},
}
)
.get(
@ -82,14 +82,14 @@ export const addSongHandler = new Elysia()
id: jobID,
state: "completed",
result: {
message: "Video already exists in the songs table."
}
message: "Video already exists in the songs table.",
},
};
}
const job = await LatestVideosQueue.getJob(jobID);
if (!job) {
return status(404, {
message: "Job not found."
message: "Job not found.",
});
}
const state = await job.getState();
@ -97,7 +97,7 @@ export const addSongHandler = new Elysia()
id: job.id!,
state,
result: job.returnvalue,
failedReason: job.failedReason
failedReason: job.failedReason,
};
},
{
@ -106,21 +106,21 @@ export const addSongHandler = new Elysia()
id: t.String(),
state: t.String(),
result: t.Optional(t.Any()),
failedReason: t.Optional(t.String())
failedReason: t.Optional(t.String()),
}),
404: t.Object({
message: t.String()
})
message: t.String(),
}),
},
params: t.Object({
id: t.String()
id: t.String(),
}),
detail: {
summary: "Check import job status",
description:
"This endpoint retrieves the current status of a song import job. It returns the job state \
(completed, failed, active, etc.), the result if completed, and any failure reason if the job failed. \
Use this endpoint to monitor the progress of song imports initiated through the import endpoint."
}
Use this endpoint to monitor the progress of song imports initiated through the import endpoint.",
},
}
);

22
packages/backend/routes/song/delete.ts
View File

@ -1,7 +1,7 @@
import { Elysia, t } from "elysia";
import { requireAuth } from "@backend/middlewares/auth";
import { songs, history, db } from "@core/drizzle";
import { db, history, songs } from "@core/drizzle";
import { eq } from "drizzle-orm";
import { Elysia, t } from "elysia";
export const deleteSongHandler = new Elysia({ prefix: "/song" }).use(requireAuth).delete(
"/:id",
@ -12,33 +12,33 @@ export const deleteSongHandler = new Elysia({ prefix: "/song" }).use(requireAuth
objectId: id,
changeType: "del-song",
changedBy: user!.unqId,
data: null
data: null,
});
return {
message: `Successfully deleted song ${id}.`
message: `Successfully deleted song ${id}.`,
};
},
{
response: {
200: t.Object({
message: t.String()
message: t.String(),
}),
401: t.Object({
message: t.String()
message: t.String(),
}),
500: t.Object({
message: t.String()
})
message: t.String(),
}),
},
params: t.Object({
id: t.String()
id: t.String(),
}),
detail: {
summary: "Delete song",
description:
"This endpoint allows authenticated users to soft-delete a song from the database. \
The song is marked as deleted rather than being permanently removed, preserving data integrity. \
The deletion is logged in the history table for audit purposes. Requires authentication and appropriate permissions."
}
The deletion is logged in the history table for audit purposes. Requires authentication and appropriate permissions.",
},
}
);

60
packages/backend/routes/song/info.ts
View File

@ -1,8 +1,8 @@
import { Elysia, t } from "elysia";
import { db, eta, history, songs, videoSnapshot } from "@core/drizzle";
import { eq, and, desc } from "drizzle-orm";
import { bv2av } from "@backend/lib/bilibiliID";
import { requireAuth } from "@backend/middlewares/auth";
import { db, eta, history, songs, videoSnapshot } from "@core/drizzle";
import { and, desc, eq } from "drizzle-orm";
import { Elysia, t } from "elysia";
async function getSongIDFromBiliID(id: string) {
let aid: number;
@ -48,11 +48,11 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
if (Number.isNaN(songID)) {
return status(404, {
code: "SONG_NOT_FOUND",
message: "Given song cannot be found."
message: "Given song cannot be found.",
});
}
return {
songID
songID,
};
})
.get(
@ -62,7 +62,7 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
if (!info) {
return status(404, {
code: "SONG_NOT_FOUND",
message: "Given song cannot be found."
message: "Given song cannot be found.",
});
}
return {
@ -72,7 +72,7 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
producer: info.producer,
duration: info.duration,
cover: info.image || undefined,
publishedAt: info.publishedAt
publishedAt: info.publishedAt,
};
},
{
@ -84,15 +84,15 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
producer: t.Union([t.String(), t.Null()]),
duration: t.Union([t.Number(), t.Null()]),
cover: t.Optional(t.String()),
publishedAt: t.Union([t.String(), t.Null()])
publishedAt: t.Union([t.String(), t.Null()]),
}),
404: t.Object({
code: t.String(),
message: t.String()
})
message: t.String(),
}),
},
headers: t.Object({
Authorization: t.Optional(t.String())
Authorization: t.Optional(t.String()),
}),
detail: {
summary: "Get information of a song",
@ -101,23 +101,23 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
which can be provided in several formats. \
The endpoint accepts a song ID in either a numerical format as the internal ID in our database\
or as a bilibili video ID (either av or BV format). \
It responds with the song's name, bilibili ID (av), producer, duration, and associated singers."
}
It responds with the song's name, bilibili ID (av), producer, duration, and associated singers.",
},
}
)
.get("/snapshots", async ({ status, songID }) => {
const r = await db.select().from(songs).where(eq(songs.id, songID)).limit(1);
if (r.length == 0) {
if (r.length === 0) {
return status(404, {
code: "SONG_NOT_FOUND",
message: "Given song cannot be found."
message: "Given song cannot be found.",
});
}
const song = r[0];
const aid = song.aid;
if (!aid) {
return status(404, {
message: "Given song is not associated with any bilibili video."
message: "Given song is not associated with any bilibili video.",
});
}
return db
@ -128,17 +128,17 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
})
.get("/eta", async ({ status, songID }) => {
const r = await db.select().from(songs).where(eq(songs.id, songID)).limit(1);
if (r.length == 0) {
if (r.length === 0) {
return status(404, {
code: "SONG_NOT_FOUND",
message: "Given song cannot be found."
message: "Given song cannot be found.",
});
}
const song = r[0];
const aid = song.aid;
if (!aid) {
return status(404, {
message: "Given song is not associated with any bilibili video."
message: "Given song is not associated with any bilibili video.",
});
}
const data = await db.select().from(eta).where(eq(eta.aid, aid));
@ -158,7 +158,7 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
if (!info) {
return status(404, {
code: "SONG_NOT_FOUND",
message: "Given song cannot be found."
message: "Given song cannot be found.",
});
}
@ -181,32 +181,32 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
updatedData.length > 0
? {
old: info,
new: updatedData[0]
new: updatedData[0],
}
: null
: null,
});
return {
message: "Successfully updated song info.",
updated: updatedData.length > 0 ? updatedData[0] : null
updated: updatedData.length > 0 ? updatedData[0] : null,
};
},
{
response: {
200: t.Object({
message: t.String(),
updated: t.Any()
updated: t.Any(),
}),
401: t.Object({
message: t.String()
message: t.String(),
}),
404: t.Object({
message: t.String(),
code: t.String()
})
code: t.String(),
}),
},
body: t.Object({
name: t.Optional(t.String()),
producer: t.Optional(t.String())
producer: t.Optional(t.String()),
}),
detail: {
summary: "Update song information",
@ -214,7 +214,7 @@ export const songHandler = new Elysia({ prefix: "/song/:id" })
"This endpoint allows authenticated users to update song metadata. It accepts partial updates \
for song name and producer fields. The endpoint validates the song ID (accepting both internal database IDs \
and bilibili video IDs in av/BV format), applies the requested changes, and logs the update in the history table \
for audit purposes. Requires authentication."
}
for audit purposes. Requires authentication.",
},
}
);

28
packages/backend/routes/song/milestone.ts
View File

@ -1,16 +1,16 @@
import { Elysia, t } from "elysia";
import { db, bilibiliMetadata, eta } from "@core/drizzle";
import { eq, and, gte, lt } from "drizzle-orm";
import serverTiming from "@backend/middlewares/timing";
import z from "zod";
import { BiliVideoSchema } from "@backend/lib/schema";
import serverTiming from "@backend/middlewares/timing";
import { bilibiliMetadata, db, eta } from "@core/drizzle";
import { and, eq, gte, lt } from "drizzle-orm";
import { Elysia, t } from "elysia";
import z from "zod";
type MileStoneType = "dendou" | "densetsu" | "shinwa";
const range = {
dendou: [0, 100000],
densetsu: [100000, 1000000],
shinwa: [1000000, 10000000]
shinwa: [1000000, 10000000],
};
export const closeMileStoneHandler = new Elysia({ prefix: "/songs" }).use(serverTiming()).get(
@ -39,21 +39,21 @@ export const closeMileStoneHandler = new Elysia({ prefix: "/songs" }).use(server
eta: z.number(),
speed: z.number(),
currentViews: z.number(),
updatedAt: z.string()
updatedAt: z.string(),
}),
bilibili_metadata: BiliVideoSchema
bilibili_metadata: BiliVideoSchema,
})
),
404: t.Object({
message: t.String()
})
message: t.String(),
}),
},
params: t.Object({
type: t.String({ enum: ["dendou", "densetsu", "shinwa"] })
type: t.String({ enum: ["dendou", "densetsu", "shinwa"] }),
}),
query: t.Object({
offset: t.Optional(t.Number()),
limit: t.Optional(t.Number())
limit: t.Optional(t.Number()),
}),
detail: {
summary: "Get songs close to milestones",
@ -61,7 +61,7 @@ export const closeMileStoneHandler = new Elysia({ prefix: "/songs" }).use(server
"This endpoint retrieves songs that are approaching significant view count milestones. \
It supports three milestone types: 'dendou' (0-100k views), 'densetsu' (100k-1M views), and 'shinwa' (1M-10M views). \
For each type, it returns videos that are within the specified view range and have an estimated time to reach \
the next milestone below the threshold. Results are ordered by estimated time to milestone."
}
the next milestone below the threshold. Results are ordered by estimated time to milestone.",
},
}
);

25
packages/backend/routes/video/eta.ts
View File

@ -1,7 +1,7 @@
import { Elysia, t } from "elysia";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { db, eta } from "@core/drizzle";
import { eq } from "drizzle-orm";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { Elysia, t } from "elysia";
export const songEtaHandler = new Elysia({ prefix: "/video" }).get(
"/:id/eta",
@ -11,14 +11,15 @@ export const songEtaHandler = new Elysia({ prefix: "/video" }).get(
if (!aid) {
return status(400, {
code: "MALFORMED_SLOT",
message: "We cannot parse the video ID, or we currently do not support this format."
message:
"We cannot parse the video ID, or we currently do not support this format.",
});
}
const data = await db.select().from(eta).where(eq(eta.aid, aid));
if (data.length === 0) {
return status(404, {
code: "VIDEO_NOT_FOUND",
message: "Video not found."
message: "Video not found.",
});
}
return {
@ -26,7 +27,7 @@ export const songEtaHandler = new Elysia({ prefix: "/video" }).get(
eta: data[0].eta,
views: data[0].currentViews,
speed: data[0].speed,
updatedAt: data[0].updatedAt
updatedAt: data[0].updatedAt,
};
},
{
@ -36,19 +37,19 @@ export const songEtaHandler = new Elysia({ prefix: "/video" }).get(
eta: t.Number(),
views: t.Number(),
speed: t.Number(),
updatedAt: t.String()
updatedAt: t.String(),
}),
400: t.Object({
code: t.String(),
message: t.String()
message: t.String(),
}),
404: t.Object({
code: t.String(),
message: t.String()
})
message: t.String(),
}),
},
headers: t.Object({
Authorization: t.Optional(t.String())
Authorization: t.Optional(t.String()),
}),
detail: {
summary: "Get video milestone ETA",
@ -56,7 +57,7 @@ export const songEtaHandler = new Elysia({ prefix: "/video" }).get(
"This endpoint retrieves the estimated time to reach the next milestone for a given video. \
It accepts video IDs in av or BV format and returns the current view count, estimated time to \
reach the next milestone (in hours), view growth speed, and last update timestamp. Useful for \
tracking video growth and milestone predictions."
}
tracking video growth and milestone predictions.",
},
}
);

28
packages/backend/routes/video/label.ts
View File

@ -1,11 +1,11 @@
import { Elysia, t } from "elysia";
import { ErrorResponseSchema } from "@backend/src/schema";
import z from "zod";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { BiliVideoSchema } from "@backend/lib/schema";
import requireAuth from "@backend/middlewares/auth";
import { eq, sql } from "drizzle-orm";
import { ErrorResponseSchema } from "@backend/src/schema";
import { bilibiliMetadata, db, videoTypeLabelInInternal } from "@core/drizzle";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { eq, sql } from "drizzle-orm";
import { Elysia, t } from "elysia";
import z from "zod";
const videoSchema = BiliVideoSchema.omit({ publishedAt: true })
.omit({ createdAt: true })
@ -16,7 +16,7 @@ const videoSchema = BiliVideoSchema.omit({ publishedAt: true })
uid: z.number(),
published_at: z.string(),
createdAt: z.string(),
cover_url: z.string()
cover_url: z.string(),
});
export const getUnlabelledVideos = new Elysia({ prefix: "/videos" }).use(requireAuth).get(
@ -60,8 +60,8 @@ export const getUnlabelledVideos = new Elysia({ prefix: "/videos" }).use(require
response: {
200: z.array(videoSchema),
400: ErrorResponseSchema,
500: ErrorResponseSchema
}
500: ErrorResponseSchema,
},
}
);
@ -77,7 +77,7 @@ export const postVideoLabel = new Elysia({ prefix: "/video" }).use(requireAuth).
code: "MALFORMED_SLOT",
message:
"We cannot parse the video ID, or we currently do not support this format.",
errors: []
errors: [],
});
}
@ -91,23 +91,23 @@ export const postVideoLabel = new Elysia({ prefix: "/video" }).use(requireAuth).
return status(400, {
code: "VIDEO_NOT_FOUND",
message: "Video not found",
errors: []
errors: [],
});
}
await db.insert(videoTypeLabelInInternal).values({
aid,
label,
user: user!.unqId
user: user!.unqId,
});
return status(201, {
message: `Labelled video av${aid} as ${label}`
message: `Labelled video av${aid} as ${label}`,
});
},
{
body: t.Object({
label: t.Boolean()
})
label: t.Boolean(),
}),
}
);

26
packages/backend/routes/video/metadata.ts
View File

@ -1,12 +1,12 @@
import { Elysia } from "elysia";
import { db, videoSnapshot } from "@core/drizzle";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { getVideoInfo } from "@core/net/getVideoInfo";
import { redis } from "@core/db/redis";
import { ErrorResponseSchema } from "@backend/src/schema";
import type { VideoInfoData } from "@core/net/bilibili.d.ts";
import { BiliAPIVideoMetadataSchema } from "@backend/lib/schema";
import { ErrorResponseSchema } from "@backend/src/schema";
import { redis } from "@core/db/redis";
import { db, videoSnapshot } from "@core/drizzle";
import type { VideoInfoData } from "@core/net/bilibili.d.ts";
import { getVideoInfo } from "@core/net/getVideoInfo";
import { snapshotCounter } from "@crawler/metrics";
import { Elysia } from "elysia";
export async function retrieveVideoInfoFromCache(aid: number) {
const cacheKey = `cvsa:videoInfo:av${aid}`;
@ -40,7 +40,7 @@ async function insertVideoSnapshot(data: VideoInfoData) {
likes,
coins,
shares,
favorites
favorites,
});
snapshotCounter.add(1);
}
@ -56,7 +56,7 @@ export const getVideoMetadataHandler = new Elysia({ prefix: "/video" }).get(
code: "MALFORMED_SLOT",
message:
"We cannot parse the video ID, or we currently do not support this format.",
errors: []
errors: [],
});
}
@ -67,11 +67,11 @@ export const getVideoMetadataHandler = new Elysia({ prefix: "/video" }).get(
const r = await getVideoInfo(aid, "getVideoInfo");
if (typeof r == "number") {
if (typeof r === "number") {
return c.status(500, {
code: "THIRD_PARTY_ERROR",
message: `Got status code ${r} from bilibili API.`,
errors: []
errors: [],
});
}
@ -86,7 +86,7 @@ export const getVideoMetadataHandler = new Elysia({ prefix: "/video" }).get(
response: {
200: BiliAPIVideoMetadataSchema,
400: ErrorResponseSchema,
500: ErrorResponseSchema
500: ErrorResponseSchema,
},
detail: {
summary: "Get video metadata",
@ -94,7 +94,7 @@ export const getVideoMetadataHandler = new Elysia({ prefix: "/video" }).get(
"This endpoint retrieves comprehensive metadata for a bilibili video. It accepts video IDs in av or BV format \
and returns detailed information including title, description, uploader, statistics (views, likes, coins, etc.), \
and publication date. The data is cached for 60 seconds to reduce API calls. If the video is not in cache, \
it fetches fresh data from bilibili API and stores a snapshot in the database."
}
it fetches fresh data from bilibili API and stores a snapshot in the database.",
},
}
);

22
packages/backend/routes/video/snapshots.ts
View File

@ -1,10 +1,10 @@
import { Elysia } from "elysia";
import { db, videoSnapshot } from "@core/drizzle";
import { biliIDToAID } from "@backend/lib/bilibiliID";
import { ErrorResponseSchema } from "@backend/src/schema";
import { eq, desc } from "drizzle-orm";
import z from "zod";
import { SnapshotQueue } from "@backend/lib/mq";
import { ErrorResponseSchema } from "@backend/src/schema";
import { db, videoSnapshot } from "@core/drizzle";
import { desc, eq } from "drizzle-orm";
import { Elysia } from "elysia";
import z from "zod";
export const getVideoSnapshotsHandler = new Elysia({ prefix: "/video" }).get(
"/:id/snapshots",
@ -17,7 +17,7 @@ export const getVideoSnapshotsHandler = new Elysia({ prefix: "/video" }).get(
code: "MALFORMED_SLOT",
message:
"We cannot parse the video ID, or we currently do not support this format.",
errors: []
errors: [],
});
}
@ -29,7 +29,7 @@ export const getVideoSnapshotsHandler = new Elysia({ prefix: "/video" }).get(
if (data.length === 0) {
await SnapshotQueue.add("directSnapshot", {
aid
aid,
});
}
@ -48,11 +48,11 @@ export const getVideoSnapshotsHandler = new Elysia({ prefix: "/video" }).get(
shares: z.number().nullable(),
danmakus: z.number().nullable(),
aid: z.number(),
replies: z.number().nullable()
replies: z.number().nullable(),
})
),
400: ErrorResponseSchema,
500: ErrorResponseSchema
500: ErrorResponseSchema,
},
detail: {
summary: "Get video snapshots",
@ -60,7 +60,7 @@ export const getVideoSnapshotsHandler = new Elysia({ prefix: "/video" }).get(
"This endpoint retrieves historical view count snapshots for a bilibili video. It accepts video IDs in av or BV format \
and returns a chronological list of snapshots showing how the video's statistics (views, likes, coins, favorites, etc.) \
have changed over time. If no snapshots exist for the video, it automatically queues a snapshot job to collect initial data. \
Results are ordered by creation date in descending order."
}
Results are ordered by creation date in descending order.",
},
}
);

38
packages/backend/src/index.ts
View File

@ -1,22 +1,22 @@
import { Elysia, ErrorHandler } from "elysia";
import { getBindingInfo, logStartup } from "./startMessage";
import { pingHandler } from "@backend/routes/ping";
import { cors } from "@elysiajs/cors";
import { songHandler } from "@backend/routes/song/info";
import { rootHandler } from "@backend/routes/root";
import { getVideoMetadataHandler } from "@backend/routes/video/metadata";
import { closeMileStoneHandler } from "@backend/routes/song/milestone";
import { authHandler } from "@backend/routes/auth";
import { onAfterHandler } from "./onAfterHandle";
import { pingHandler } from "@backend/routes/ping";
import { rootHandler } from "@backend/routes/root";
import { searchHandler } from "@backend/routes/search";
import { getVideoSnapshotsHandler } from "@backend/routes/video/snapshots";
import { addSongHandler } from "@backend/routes/song/add";
import { deleteSongHandler } from "@backend/routes/song/delete";
import { songHandler } from "@backend/routes/song/info";
import { closeMileStoneHandler } from "@backend/routes/song/milestone";
import { songEtaHandler } from "@backend/routes/video/eta";
import { getVideoMetadataHandler } from "@backend/routes/video/metadata";
import { getVideoSnapshotsHandler } from "@backend/routes/video/snapshots";
import { cors } from "@elysiajs/cors";
import { Elysia, type ErrorHandler } from "elysia";
import { onAfterHandler } from "./onAfterHandle";
import { getBindingInfo, logStartup } from "./startMessage";
import "./mq";
import pkg from "../package.json";
import { getUnlabelledVideos, postVideoLabel } from "@backend/routes/video/label";
import { openAPIMiddleware } from "@backend/middlewares/openapi";
import { getUnlabelledVideos, postVideoLabel } from "@backend/routes/video/label";
import pkg from "../package.json";
const [host, port] = getBindingInfo();
logStartup(host, port);
@ -24,7 +24,7 @@ logStartup(host, port);
const errorHandler: ErrorHandler = ({ code, status, error }) => {
if (code === "NOT_FOUND")
return status(404, {
message: "The requested resource was not found."
message: "The requested resource was not found.",
});
if (code === "VALIDATION") return error.detail(error.message);
return error;
@ -32,8 +32,8 @@ const errorHandler: ErrorHandler = ({ code, status, error }) => {
const app = new Elysia({
serve: {
hostname: host
}
hostname: host,
},
})
.onError(errorHandler)
.use(onAfterHandler)
@ -60,8 +60,8 @@ const app = new Elysia({
},
{
detail: {
hide: true
}
hide: true,
},
}
)
.get(
@ -71,8 +71,8 @@ const app = new Elysia({
},
{
detail: {
hide: true
}
hide: true,
},
}
)
.listen(15412);

6
packages/backend/src/mq.ts
View File

@ -1,12 +1,12 @@
import { db, history } from "@core/drizzle";
import { ConnectionOptions, QueueEvents, QueueEventsListener } from "bullmq";
import { type ConnectionOptions, QueueEvents, type QueueEventsListener } from "bullmq";
import { redis } from "bun";
interface CustomListener extends QueueEventsListener {
addSong: (args: { uid: string; songID: number }, id: string) => void;
}
const queueEvents = new QueueEvents("latestVideos", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
queueEvents.on<CustomListener>(
"addSong",
@ -15,7 +15,7 @@ queueEvents.on<CustomListener>(
objectId: songID,
changeType: "add-song",
changedBy: uid,
data: null
data: null,
});
}
);

8
packages/backend/src/onAfterHandle.ts
View File

@ -26,8 +26,8 @@ export const onAfterHandler = new Elysia().onAfterHandle(
return new Response(encoder.encode(text), {
status: realResponse.code as any,
headers: {
"Content-Type": "application/json; charset=utf-8"
}
"Content-Type": "application/json; charset=utf-8",
},
});
}
const text = isBrowser
@ -35,8 +35,8 @@ export const onAfterHandler = new Elysia().onAfterHandle(
: JSON.stringify(realResponse);
return new Response(encoder.encode(text), {
headers: {
"Content-Type": "application/json; charset=utf-8"
}
"Content-Type": "application/json; charset=utf-8",
},
});
}
);

6
packages/backend/src/schema.ts
View File

@ -13,7 +13,7 @@ export const errorCodes = [
"SERVER_ERROR",
"RATE_LIMIT_EXCEEDED",
"ENTITY_EXISTS",
"THIRD_PARTY_ERROR"
"THIRD_PARTY_ERROR",
];
function generateErrorCodeRegex(strings: string[]): string {
@ -33,7 +33,7 @@ export const ErrorResponseSchema = t.Object({
i18n: t.Optional(
t.Object({
key: t.String(),
values: t.Optional(t.Record(t.String(), t.Union([t.String(), t.Number(), t.Date()])))
values: t.Optional(t.Record(t.String(), t.Union([t.String(), t.Number(), t.Date()]))),
})
)
),
});

2
packages/backend/src/startMessage.ts
View File

@ -1,5 +1,5 @@
import os from "os";
import chalk from "chalk";
import os from "os";
function getLocalIpAddress(): string {
const interfaces = os.networkInterfaces();

10
packages/core/biome.json Normal file
View File

@ -0,0 +1,10 @@
{
"extends": "//",
"$schema": "https://biomejs.dev/schemas/2.3.8/schema.json",
"files": {
"includes": ["**", "!!**/drizzle/main"]
},
"linter": {
"enabled": true
}
}

2
packages/core/db/dbNew.ts
View File

@ -1,4 +1,4 @@
import postgres from "postgres";
import { postgresConfig } from "./pgConfigNew";
export const sql = postgres(postgresConfig);
export const sql = postgres(postgresConfig);

4
packages/core/db/pgConfigNew.ts
View File

@ -22,7 +22,7 @@ export const postgresConfig = {
port: parseInt(databasePort),
database: databaseName,
username: databaseUser,
password: databasePassword
password: databasePassword,
};
export const postgresConfigCred = {
@ -30,5 +30,5 @@ export const postgresConfigCred = {
port: parseInt(databasePort),
database: databaseNameCred,
user: databaseUser,
password: databasePassword
password: databasePassword,
};

2
packages/core/db/redis.ts
View File

@ -6,5 +6,5 @@ const port = parseInt(process.env.REDIS_PORT) || 6379;
export const redis = new Redis({
port: port,
host: host,
maxRetriesPerRequest: null
maxRetriesPerRequest: null,
});

2
packages/core/db/snapshots/index.ts
View File

@ -1,3 +1,3 @@
export * from "./getLatestSnapshot";
export * from "./getClosetSnapshot";
export * from "./getLatestSnapshot";
export * from "./milestone";

4
packages/core/db/snapshots/milestone.ts
View File

@ -1,7 +1,7 @@
import { getClosestSnapshot, getLatestSnapshot } from "@core/db";
import { db, eta as etaTable } from "@core/drizzle";
import { getClosetMilestone, HOUR, MINUTE } from "@core/lib";
import { eq } from "drizzle-orm";
import { MINUTE, HOUR, getClosetMilestone } from "@core/lib";
import { getLatestSnapshot, getClosestSnapshot } from "@core/db";
export const getGroundTruthMilestoneETA = async (
aid: number,

4
packages/core/drizzle.config.ts
View File

@ -5,7 +5,7 @@ export default defineConfig({
out: "./drizzle/main",
dialect: "postgresql",
dbCredentials: {
url: process.env.DATABASE_URL_MAIN!
url: process.env.DATABASE_URL_MAIN!,
},
schemaFilter: ["public", "credentials", "internal"]
schemaFilter: ["public", "credentials", "internal"],
});

4
packages/core/drizzle/drizzle-cred.config.ts
View File

@ -4,6 +4,6 @@ export default defineConfig({
out: "./cred",
dialect: "postgresql",
dbCredentials: {
url: process.env.DATABASE_URL_CRED!
}
url: process.env.DATABASE_URL_CRED!,
},
});

8
packages/core/drizzle/drizzle-main.config.ts
View File

@ -1,10 +1,14 @@
import "dotenv/config";
import { defineConfig } from "drizzle-kit";
if (!process.env.DATABASE_URL_MAIN) {
throw new Error("DATABASE_URL_MAIN is not defined");
}
export default defineConfig({
out: "./drizzle/main",
dialect: "postgresql",
dbCredentials: {
url: process.env.DATABASE_URL_MAIN!
}
url: process.env.DATABASE_URL_MAIN,
},
});

4
packages/core/drizzle/index.ts
View File

@ -1,8 +1,8 @@
"use server";
import { drizzle } from "drizzle-orm/postgres-js";
import { sql } from "@core/db/dbNew";
import { drizzle } from "drizzle-orm/postgres-js";
export const db = drizzle(sql);
export * from "./main/schema";
export * from "./type";
export * from "./type";

10
packages/core/drizzle/type.ts
View File

@ -1,12 +1,12 @@
import type { InferSelectModel } from "drizzle-orm";
import {
usersInCredentials,
import type {
bilibiliMetadata,
latestVideoSnapshot,
songs,
videoSnapshot,
loginSessionsInCredentials,
producer,
loginSessionsInCredentials
songs,
usersInCredentials,
videoSnapshot,
} from "./main/schema";
export type UserType = InferSelectModel<typeof usersInCredentials>;

1
packages/core/index.ts
View File

@ -1 +0,0 @@
export * from "./db/dbNew";

2
packages/core/lib/index.ts
View File

@ -2,4 +2,4 @@ export * from "./math";
export * from "./milestone";
export * from "./randomID";
export * from "./time";
export * from "./type";
export * from "./type";

3
packages/core/lib/math.ts
View File

@ -1,3 +1,4 @@
export const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
export const truncate = (num: number, min: number, max: number) => Math.max(min, Math.min(num, max));
export const truncate = (num: number, min: number, max: number) =>
Math.max(min, Math.min(num, max));

2
packages/core/lib/type.ts
View File

@ -1 +1 @@
export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;
export type PartialBy<T, K extends keyof T> = Omit<T, K> & Partial<Pick<T, K>>;

22
packages/core/log/index.ts
View File

@ -1,6 +1,6 @@
import winston, { format, transports } from "winston";
import type { TransformableInfo } from "logform";
import chalk from "chalk";
import type { TransformableInfo } from "logform";
import winston, { format, transports } from "winston";
const customFormat = format.printf((info: TransformableInfo) => {
const { timestamp, level, message, service, codePath, error } = info;
@ -21,9 +21,9 @@ const timestampFormat = format.timestamp({ format: "YYYY-MM-DD HH:mm:ss.SSSZZ" }
const createTransport = (level: string, filename: string) => {
const MB = 1000000;
let maxsize = undefined;
let maxFiles = undefined;
let tailable = undefined;
let maxsize;
let maxFiles;
let tailable;
if (level === "silly") {
maxsize = 500 * MB;
maxFiles = undefined;
@ -37,7 +37,7 @@ const createTransport = (level: string, filename: string) => {
if (typeof value === "bigint") {
return value.toString();
}
if (key == "error") {
if (key === "error") {
return undefined;
}
return value;
@ -48,7 +48,7 @@ const createTransport = (level: string, filename: string) => {
maxsize,
tailable,
maxFiles,
format: format.combine(timestampFormat, format.json({ replacer }))
format: format.combine(timestampFormat, format.json({ replacer })),
});
};
@ -66,12 +66,12 @@ const winstonLogger = winston.createLogger({
format.colorize(),
format.errors({ stack: true }),
customFormat
)
),
}),
createTransport("silly", sillyLogPath),
createTransport("warn", warnLogPath),
createTransport("error", errorLogPath)
]
createTransport("error", errorLogPath),
],
});
const logger = {
@ -96,7 +96,7 @@ const logger = {
} else {
winstonLogger.error(error, { service, codePath });
}
}
},
};
export default logger;

2
packages/core/mq/lockManager.ts
View File

@ -1,5 +1,5 @@
import { Redis } from "ioredis";
import { redis } from "@core/db/redis";
import type { Redis } from "ioredis";
class LockManager {
private redis: Redis;

4
packages/core/mq/multipleRateLimiter.ts
View File

@ -1,5 +1,5 @@
import { RateLimiter as Limiter } from "@koshnic/ratelimit";
import { redis } from "@core/db/redis";
import { RateLimiter as Limiter } from "@koshnic/ratelimit";
export interface RateLimiterConfig {
duration: number;
@ -42,7 +42,7 @@ export class MultipleRateLimiter {
burst: max,
ratePerPeriod: max,
period: duration,
cost: 1
cost: 1,
});
if (!allowed && shouldThrow) {
throw new RateLimiterError("Rate limit exceeded");

72
packages/core/net/delegate.ts
View File

@ -1,25 +1,25 @@
// noinspection ExceptionCaughtLocallyJS
import Credential from "@alicloud/credentials";
import Stream from "@alicloud/darabonba-stream";
import FC20230330, * as $FC20230330 from "@alicloud/fc20230330";
import * as OpenApi from "@alicloud/openapi-client";
import * as Util from "@alicloud/tea-util";
import { SECOND } from "@core/lib";
import logger from "@core/log";
import {
MultipleRateLimiter,
type RateLimiterConfig,
RateLimiterError
RateLimiterError,
} from "@core/mq/multipleRateLimiter";
import { ReplyError } from "ioredis";
import { SECOND } from "@core/lib";
import FC20230330, * as $FC20230330 from "@alicloud/fc20230330";
import Credential from "@alicloud/credentials";
import * as OpenApi from "@alicloud/openapi-client";
import Stream from "@alicloud/darabonba-stream";
import * as Util from "@alicloud/tea-util";
import { Readable } from "stream";
import {
aliFCCounter,
aliFCErrorCounter,
ipProxyCounter,
ipProxyErrorCounter
ipProxyErrorCounter,
} from "crawler/metrics";
import { ReplyError } from "ioredis";
import type { Readable } from "stream";
type ProxyType = "native" | "alicloud-fc" | "ip-proxy";
@ -33,8 +33,8 @@ function createAliProxiesObject<T extends readonly string[]>(regions: T) {
type: "alicloud-fc" as const,
data: {
region: currentRegion,
timeout: 15000
}
timeout: 15000,
},
} as ProxyDef<AlicloudFcProxyData>;
return result;
},
@ -48,7 +48,7 @@ const aliProxies = aliRegions.map((region) => `alicloud_${region}` as `alicloud_
const proxies = {
native: {
type: "native" as const,
data: {}
data: {},
},
...aliProxiesObject,
@ -79,7 +79,7 @@ const proxies = {
port: item.port,
lifespan: Date.parse(item.endtime + "+08") - Date.now(),
createdAt: Date.now(),
used: false
used: false,
};
});
},
@ -87,9 +87,9 @@ const proxies = {
minPoolSize: 10,
maxPoolSize: 100,
refreshInterval: 5 * SECOND,
initialPoolSize: 10
}
}
initialPoolSize: 10,
},
},
} satisfies Record<string, ProxyDef>;
interface FCResponse {
@ -98,7 +98,7 @@ interface FCResponse {
serverTime: number;
}
interface NativeProxyData {}
type NativeProxyData = {};
interface AlicloudFcProxyData {
region: (typeof aliRegions)[number];
@ -169,7 +169,7 @@ interface NetworkConfigInternal<ProviderKeys extends string> {
const biliLimiterConfig: RateLimiterConfig[] = [
{ duration: 1, max: 20 },
{ duration: 15, max: 130 },
{ duration: 5 * 60, max: 2000 }
{ duration: 5 * 60, max: 2000 },
];
const bili_normal = structuredClone(biliLimiterConfig);
@ -196,40 +196,40 @@ const config = createNetworkConfig({
proxies: proxies,
providers: {
test: { limiters: [] },
bilibili: { limiters: [] }
bilibili: { limiters: [] },
},
tasks: {
test: {
provider: "test",
proxies: fcProxies
proxies: fcProxies,
},
test_ip: {
provider: "test",
proxies: ["ip_proxy_pool"]
proxies: ["ip_proxy_pool"],
},
getVideoInfo: {
provider: "bilibili",
proxies: "all",
limiters: bili_strict
limiters: bili_strict,
},
getLatestVideos: {
provider: "bilibili",
proxies: "all",
limiters: bili_strict
limiters: bili_strict,
},
snapshotMilestoneVideo: {
provider: "bilibili",
proxies: aliProxies
proxies: aliProxies,
},
snapshotVideo: {
provider: "bilibili",
proxies: aliProxies
proxies: aliProxies,
},
bulkSnapshot: {
provider: "bilibili",
proxies: aliProxies
}
}
proxies: aliProxies,
},
},
});
type NetworkConfig = typeof config;
@ -279,7 +279,7 @@ class IPPoolManager {
minPoolSize: config.minPoolSize ?? 5,
maxPoolSize: config.maxPoolSize ?? 50,
refreshInterval: config.refreshInterval ?? 30_000,
initialPoolSize: config.initialPoolSize ?? 10
initialPoolSize: config.initialPoolSize ?? 10,
};
}
@ -374,7 +374,7 @@ class IPPoolManager {
const ipEntry: IPEntry = {
...ipData,
createdAt: Date.now(),
used: false
used: false,
};
this.pool.push(ipEntry);
}
@ -458,14 +458,14 @@ export class NetworkDelegate<const C extends NetworkConfig> {
this.tasks[taskName] = {
provider: taskDef.provider,
proxies: [...targetProxies]
proxies: [...targetProxies],
};
if (taskDef.limiters && taskDef.limiters.length > 0) {
for (const proxyName of targetProxies) {
const limiterId = `proxy-${proxyName}-${taskName}`;
this.proxyLimiters[limiterId] = new MultipleRateLimiter(limiterId, [
...taskDef.limiters
...taskDef.limiters,
]);
}
}
@ -485,7 +485,7 @@ export class NetworkDelegate<const C extends NetworkConfig> {
const limiterId = `provider-${proxyName}-${providerName}`;
if (!this.providerLimiters[limiterId]) {
this.providerLimiters[limiterId] = new MultipleRateLimiter(limiterId, [
...providerDef.limiters
...providerDef.limiters,
]);
}
}
@ -654,7 +654,7 @@ export class NetworkDelegate<const C extends NetworkConfig> {
} else {
return {
data: JSON.parse(rawData.body) as R,
time: rawData.serverTime
time: rawData.serverTime,
};
}
} catch (e) {
@ -700,7 +700,7 @@ export class NetworkDelegate<const C extends NetworkConfig> {
const response = await fetch(url, {
signal: controller.signal,
proxy: `http://${ipEntry.address}:${ipEntry.port}`
proxy: `http://${ipEntry.address}:${ipEntry.port}`,
});
clearTimeout(timeout);

4
packages/core/net/getVideoDetails.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate";
import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d";
import networkDelegate from "@core/net/delegate";
export async function getVideoDetails(aid: number): Promise<VideoDetailsData | null> {
const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`;

6
packages/core/net/getVideoInfo.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate";
import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d";
import networkDelegate from "@core/net/delegate";
/*
* Fetch video metadata from bilibili API
@ -34,6 +34,6 @@ export async function getVideoInfo(
}
return {
data: data.data,
time: time
time: time,
};
}

12
packages/core/net/services.ts
View File

@ -1,19 +1,19 @@
import type { VideoSnapshotType } from "@core/drizzle";
import type { PartialBy } from "@core/lib";
import type { VideoInfoResponse } from "@core/net/bilibili";
import networkDelegate, { type RequestTasks } from "@core/net/delegate.ts";
import { VideoInfoResponse } from "@core/net/bilibili";
import { PartialBy } from "@core/lib";
import { VideoSnapshotType } from "@core/drizzle";
export class BilibiliService {
private static videoMetadataUrl = "https://api.bilibili.com/x/web-interface/view";
private static async getVideoMetadata(aid: number, task: RequestTasks) {
const url = new URL(this.videoMetadataUrl);
const url = new URL(BilibiliService.videoMetadataUrl);
url.searchParams.set("aid", aid.toString());
return networkDelegate.request<VideoInfoResponse>(url.toString(), task);
}
static async milestoneSnapshot(aid: number): Promise<PartialBy<VideoSnapshotType, "id">> {
const metadata = await this.getVideoMetadata(aid, "snapshotMilestoneVideo");
const metadata = await BilibiliService.getVideoMetadata(aid, "snapshotMilestoneVideo");
const stats = metadata.data.data.stat;
return {
aid,
@ -24,7 +24,7 @@ export class BilibiliService {
favorites: stats.favorite,
replies: stats.reply,
shares: stats.share,
danmakus: stats.danmaku
danmakus: stats.danmaku,
};
}
}

54
packages/core/package.json
View File

@ -1,31 +1,27 @@
{
"name": "core",
"private": false,
"version": "0.0.10",
"scripts": {
"test": "bun --env-file=.env.test run vitest",
"build": "bun build ./index.ts --target node --outdir ./dist",
"drizzle:pull": "drizzle-kit pull"
},
"dependencies": {
"@alicloud/credentials": "^2.4.4",
"@alicloud/darabonba-stream": "^0.0.2",
"@alicloud/fc20230330": "^4.6.2",
"@alicloud/openapi-client": "^0.4.15",
"@alicloud/tea-util": "^1.4.10",
"@koshnic/ratelimit": "^1.0.3",
"@types/luxon": "^3.7.1",
"chalk": "^5.4.1",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"luxon": "^3.7.2",
"postgres": "^3.4.5",
"winston": "^3.17.0"
},
"devDependencies": {
"@types/ioredis": "^5.0.0",
"drizzle-kit": "^0.31.4"
},
"main": "./dist/index.js",
"types": "./types.d.ts"
"name": "core",
"private": true,
"scripts": {
"test": "bun --env-file=.env.test run vitest",
"drizzle:pull": "drizzle-kit pull"
},
"dependencies": {
"@alicloud/credentials": "^2.4.4",
"@alicloud/darabonba-stream": "^0.0.2",
"@alicloud/fc20230330": "^4.6.2",
"@alicloud/openapi-client": "^0.4.15",
"@alicloud/tea-util": "^1.4.10",
"@koshnic/ratelimit": "^1.0.3",
"@types/luxon": "^3.7.1",
"chalk": "^5.4.1",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"luxon": "^3.7.2",
"postgres": "^3.4.5",
"winston": "^3.17.0"
},
"devDependencies": {
"@types/ioredis": "^5.0.0",
"drizzle-kit": "^0.31.4"
}
}

11
packages/core/test/netDelegate.test.ts
View File

@ -1,15 +1,12 @@
import { describe, expect, test } from "bun:test";
import networkDelegate from "@core/net/delegate";
import { test, expect, describe } from "bun:test";
describe("proxying requests", () => {
test("Alibaba Cloud FC", async () => {
const { data } = (await networkDelegate.request<{
const { data } = await networkDelegate.request<{
headers: Record<string, string>;
}>(
"https://postman-echo.com/get",
"test"
));
expect(data.headers.referer).toBe('https://www.bilibili.com/');
}>("https://postman-echo.com/get", "test");
expect(data.headers.referer).toBe("https://www.bilibili.com/");
});
test("IP Proxy", async () => {
const { data } = await networkDelegate.request<{

3
packages/core/types.d.ts vendored
View File

@ -1,3 +0,0 @@
export * from "./db/schema";
export * from "./index";
export * from "./net/bilibili";

16
packages/crawler/db/bilibili_metadata.ts
View File

@ -1,13 +1,13 @@
import {
type BilibiliMetadataType,
bilibiliMetadata,
BilibiliMetadataType,
bilibiliUser,
db,
labellingResult
labellingResult,
} from "@core/drizzle";
import { AkariModelVersion } from "ml/const";
import type { PartialBy } from "@core/lib";
import { eq, isNull } from "drizzle-orm";
import { PartialBy } from "@core/lib";
import { AkariModelVersion } from "ml/const";
export async function insertIntoMetadata(
data: PartialBy<BilibiliMetadataType, "id" | "createdAt" | "status">
@ -18,7 +18,7 @@ export async function insertIntoMetadata(
export async function videoExistsInAllData(aid: number) {
const rows = await db
.select({
id: bilibiliMetadata.id
id: bilibiliMetadata.id,
})
.from(bilibiliMetadata)
.where(eq(bilibiliMetadata.aid, aid))
@ -53,10 +53,10 @@ export async function insertVideoLabel(aid: number, label: number) {
.values({
aid,
label,
modelVersion: AkariModelVersion
modelVersion: AkariModelVersion,
})
.onConflictDoNothing({
target: [labellingResult.aid, labellingResult.modelVersion]
target: [labellingResult.aid, labellingResult.modelVersion],
});
}
@ -75,7 +75,7 @@ export async function getVideoInfoFromAllData(aid: number) {
return {
title: row.title,
description: row.description,
tags: row.tags
tags: row.tags,
};
}

2
packages/crawler/db/eta.ts
View File

@ -1,4 +1,4 @@
import { Psql } from "@core/db/psql";
import type { Psql } from "@core/db/psql";
export async function updateETA(sql: Psql, aid: number, eta: number, speed: number, views: number) {
return sql`

15
packages/crawler/db/snapshot.ts
View File

@ -1,9 +1,14 @@
import { SnapshotNumber } from "mq/task/getVideoStats";
import type { Psql } from "@core/db/psql.d";
import { db, LatestVideoSnapshotType, videoSnapshot, VideoSnapshotType } from "@core/drizzle";
import { PartialBy } from "@core/lib";
import {
db,
type LatestVideoSnapshotType,
type VideoSnapshotType,
videoSnapshot,
} from "@core/drizzle";
import type { PartialBy } from "@core/lib";
import { sql } from "drizzle-orm";
import { snapshotCounter } from "metrics";
import type { SnapshotNumber } from "mq/task/getVideoStats";
export async function insertVideoSnapshot(data: PartialBy<VideoSnapshotType, "id">) {
await db.insert(videoSnapshot).values(data);
@ -36,7 +41,7 @@ export async function getVideosNearMilestone() {
return results.map((row) => {
return {
...row,
aid: Number(row.aid)
aid: Number(row.aid),
};
});
}
@ -57,7 +62,7 @@ export async function getLatestVideoSnapshot(
return {
...row,
aid: Number(row.aid),
time: new Date(row.time).getTime()
time: new Date(row.time).getTime(),
};
})[0];
}

18
packages/crawler/db/snapshotSchedule.ts
View File

@ -1,10 +1,10 @@
import type { SnapshotScheduleType } from "@core/db/schema.d";
import logger from "@core/log";
import { MINUTE } from "@core/lib";
import { redis } from "@core/db/redis";
import { Redis } from "ioredis";
import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre";
import type { Psql } from "@core/db/psql.d";
import { redis } from "@core/db/redis";
import type { SnapshotScheduleType } from "@core/db/schema.d";
import { MINUTE } from "@core/lib";
import logger from "@core/log";
import type { Redis } from "ioredis";
import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre";
const REDIS_KEY = "cvsa:snapshot_window_counts";
@ -141,7 +141,7 @@ export async function findClosestSnapshot(
const row = result[0];
return {
created_at: new Date(row.created_at).getTime(),
views: row.views
views: row.views,
};
}
@ -162,7 +162,7 @@ export async function findSnapshotBefore(
const row = result[0];
return {
created_at: new Date(row.created_at).getTime(),
views: row.views
views: row.views,
};
}
@ -190,7 +190,7 @@ export async function getLatestSnapshot(sql: Psql, aid: number): Promise<Snapsho
const row = res[0];
return {
created_at: new Date(row.created_at).getTime(),
views: row.views
views: row.views,
};
}

24
packages/crawler/metrics/index.ts
View File

@ -1,50 +1,50 @@
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-grpc";
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
const exporter = new OTLPMetricExporter({
url: "http://localhost:4317"
url: "http://localhost:4317",
});
const metricReader = new PeriodicExportingMetricReader({
exporter: exporter,
exportIntervalMillis: 2000
exportIntervalMillis: 2000,
});
const meterProvider = new MeterProvider({
readers: [metricReader]
readers: [metricReader],
});
const meter = meterProvider.getMeter("bullmq-worker");
const anotherMeter = meterProvider.getMeter("networking");
export const ipProxyCounter = anotherMeter.createCounter("ip_proxy_count", {
description: "Number of requests using IP proxy"
description: "Number of requests using IP proxy",
});
export const ipProxyErrorCounter = anotherMeter.createCounter("ip_proxy_error_count", {
description: "Number of errors thrown by IP proxy"
description: "Number of errors thrown by IP proxy",
});
export const aliFCCounter = anotherMeter.createCounter("ali_fc_count", {
description: "Number of requests using Ali FC"
description: "Number of requests using Ali FC",
});
export const aliFCErrorCounter = anotherMeter.createCounter("ali_fc_error_count", {
description: "Number of errors thrown by Ali FC"
description: "Number of errors thrown by Ali FC",
});
export const jobCounter = meter.createCounter("job_count", {
description: "Number of executed BullMQ jobs"
description: "Number of executed BullMQ jobs",
});
export const queueJobsCounter = meter.createGauge("queue_jobs_count", {
description: "Number of jobs in specific BullMQ queue"
description: "Number of jobs in specific BullMQ queue",
});
export const jobDurationRaw = meter.createGauge("job_duration_raw", {
description: "Execution duration of BullMQ jobs in milliseconds"
description: "Execution duration of BullMQ jobs in milliseconds",
});
export const snapshotCounter = meter.createCounter("snapshot_count", {
description: "Number of snapshots taken"
description: "Number of snapshots taken",
});

24
packages/crawler/ml/akari.ts
View File

@ -1,10 +1,10 @@
import { AIManager } from "ml/manager";
import * as ort from "onnxruntime-node";
import logger from "@core/log";
import { WorkerError } from "mq/schema";
import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers";
import { AkariModelVersion } from "./const";
import path from "node:path";
import logger from "@core/log";
import { AutoTokenizer, type PreTrainedTokenizer } from "@huggingface/transformers";
import { AIManager } from "ml/manager";
import { WorkerError } from "mq/schema";
import * as ort from "onnxruntime-node";
import { AkariModelVersion } from "./const";
const currentDir = import.meta.dir;
const modelDir = path.join(currentDir, "../../../model/");
@ -21,7 +21,7 @@ class AkariProto extends AIManager {
super();
this.models = {
classifier: onnxClassifierPath,
embedding: onnxEmbeddingPath
embedding: onnxEmbeddingPath,
};
}
@ -59,7 +59,7 @@ class AkariProto extends AIManager {
const { input_ids } = await tokenizer(texts, {
add_special_tokens: false,
return_tensor: false
return_tensor: false,
});
const cumsum = (arr: number[]): number[] =>
@ -70,17 +70,17 @@ class AkariProto extends AIManager {
const offsets: number[] = [
0,
...cumsum(input_ids.slice(0, -1).map((x: string) => x.length))
...cumsum(input_ids.slice(0, -1).map((x: string) => x.length)),
];
const flattened_input_ids = input_ids.flat();
const inputs = {
input_ids: new ort.Tensor("int64", new BigInt64Array(flattened_input_ids.map(BigInt)), [
flattened_input_ids.length
flattened_input_ids.length,
]),
offsets: new ort.Tensor("int64", new BigInt64Array(offsets.map(BigInt)), [
offsets.length
])
offsets.length,
]),
};
const { embeddings } = await session.run(inputs);

2
packages/crawler/ml/akari_api.ts
View File

@ -1,6 +1,6 @@
import apiManager from "./api_manager";
import logger from "@core/log";
import { WorkerError } from "mq/schema";
import apiManager from "./api_manager";
class AkariAPI {
private readonly serviceReady: Promise<boolean>;

14
packages/crawler/ml/api_manager.ts
View File

@ -33,9 +33,9 @@ export class APIManager {
const response = await fetch(`${this.baseUrl}/health`, {
method: "GET",
headers: {
"Content-Type": "application/json"
"Content-Type": "application/json",
},
signal: AbortSignal.timeout(this.timeout)
signal: AbortSignal.timeout(this.timeout),
});
if (!response.ok) {
@ -60,17 +60,17 @@ export class APIManager {
title: title.trim() || "untitled",
description: description.trim() || "N/A",
tags: tags.trim() || "empty",
aid: aid
aid: aid,
};
try {
const response = await fetch(`${this.baseUrl}/classify`, {
method: "POST",
headers: {
"Content-Type": "application/json"
"Content-Type": "application/json",
},
body: JSON.stringify(request),
signal: AbortSignal.timeout(this.timeout)
signal: AbortSignal.timeout(this.timeout),
});
if (!response.ok) {
@ -100,10 +100,10 @@ export class APIManager {
const response = await fetch(`${this.baseUrl}/classify_batch`, {
method: "POST",
headers: {
"Content-Type": "application/json"
"Content-Type": "application/json",
},
body: JSON.stringify(requests),
signal: AbortSignal.timeout(this.timeout * 2) // Longer timeout for batch
signal: AbortSignal.timeout(this.timeout * 2), // Longer timeout for batch
});
if (!response.ok) {

2
packages/crawler/ml/manager.ts
View File

@ -1,6 +1,6 @@
import * as ort from "onnxruntime-node";
import logger from "@core/log";
import { WorkerError } from "mq/schema";
import * as ort from "onnxruntime-node";
export class AIManager {
public sessions: { [key: string]: ort.InferenceSession } = {};

30
packages/crawler/mq/exec/archiveSnapshots.ts
View File

@ -1,20 +1,20 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { MINUTE } from "@core/lib";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import type { Job } from "bullmq";
import {
formatDistanceStrict,
formatDuration,
intervalToDuration,
nextMonday,
nextSaturday,
} from "date-fns";
import {
getCommonArchiveAids,
getVideosWithoutActiveSnapshotScheduleByType,
scheduleSnapshot
scheduleSnapshot,
} from "db/snapshotSchedule";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { MINUTE } from "@core/lib";
import { sql } from "@core/db/dbNew";
import {
nextMonday,
nextSaturday,
formatDistanceStrict,
intervalToDuration,
formatDuration
} from "date-fns";
function randomTimestampBetween(start: Date, end: Date) {
const startMs = start.getTime();
@ -43,7 +43,7 @@ export const archiveSnapshotsWorker = async (_job: Job) => {
const now = Date.now();
const date = new Date();
const formatted = formatDistanceStrict(date, nextSaturday(date).getTime(), {
unit: "hour"
unit: "hour",
});
logger.log(
`Scheduled archive snapshot for aid ${aid} in ${formatted}.`,
@ -62,7 +62,7 @@ export const archiveSnapshotsWorker = async (_job: Job) => {
const targetTime = getRandomTimeInNextWeek();
const interval = intervalToDuration({
start: new Date(),
end: new Date(targetTime)
end: new Date(targetTime),
});
const formatted = formatDuration(interval, { format: ["days", "hours"] });

24
packages/crawler/mq/exec/classifyVideo.ts
View File

@ -1,18 +1,18 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { MINUTE } from "@core/lib";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import type { Job } from "bullmq";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { aidExistsInSongs } from "db/songs";
import Akari from "ml/akari_api";
import { ClassifyVideoQueue } from "mq/index";
import { insertIntoSongs } from "mq/task/collectSongs";
import {
getUnlabelledVideos,
getVideoInfoFromAllData,
insertVideoLabel
insertVideoLabel,
} from "../../db/bilibili_metadata";
import Akari from "ml/akari_api";
import { ClassifyVideoQueue } from "mq/index";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { aidExistsInSongs } from "db/songs";
import { insertIntoSongs } from "mq/task/collectSongs";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { MINUTE } from "@core/lib";
import { sql } from "@core/db/dbNew";
export const classifyVideoWorker = async (job: Job) => {
const aid = job.data.aid;
@ -44,7 +44,7 @@ export const classifyVideoWorker = async (job: Job) => {
await job.updateData({
...job.data,
label: label
label: label,
});
return 0;

2
packages/crawler/mq/exec/collectSongs.ts
View File

@ -1,4 +1,4 @@
import { Job } from "bullmq";
import type { Job } from "bullmq";
import { collectSongs } from "mq/task/collectSongs";
export const collectSongsWorker = async (_job: Job): Promise<void> => {

4
packages/crawler/mq/exec/directSnapshot.ts
View File

@ -1,7 +1,7 @@
import { Job } from "bullmq";
import { takeVideoSnapshot } from "mq/task/getVideoStats";
import { sql } from "@core/db/dbNew";
import { lockManager } from "@core/mq/lockManager";
import type { Job } from "bullmq";
import { takeVideoSnapshot } from "mq/task/getVideoStats";
export const directSnapshotWorker = async (job: Job): Promise<void> => {
const lock = await lockManager.isLocked(`directSnapshot-${job.data.aid}`);

12
packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts
View File

@ -1,12 +1,12 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import logger from "@core/log";
import type { Job } from "bullmq";
import { getVideosNearMilestone } from "db/snapshot";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { jobCounter, jobDurationRaw } from "metrics";
import { getAdjustedShortTermETA } from "mq/scheduling";
import { truncate } from "utils/truncate";
import { scheduleSnapshot } from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { sql } from "@core/db/dbNew";
import { jobCounter, jobDurationRaw } from "metrics";
export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {
const start = Date.now();

14
packages/crawler/mq/exec/dispatchRegularSnapshots.ts
View File

@ -1,15 +1,15 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { HOUR, MINUTE, WEEK } from "@core/lib";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import type { Job } from "bullmq";
import { getLatestVideoSnapshot } from "db/snapshot";
import { truncate } from "utils/truncate";
import {
getVideosWithoutActiveSnapshotScheduleByType,
scheduleSnapshot
scheduleSnapshot,
} from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, WEEK } from "@core/lib";
import { lockManager } from "@core/mq/lockManager";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval";
import { sql } from "@core/db/dbNew";
import { truncate } from "utils/truncate";
export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise<void> => {
try {

10
packages/crawler/mq/exec/executors.ts
View File

@ -1,10 +1,10 @@
export * from "./getLatestVideos";
export * from "./getVideoInfo";
export * from "./collectSongs";
export * from "./takeBulkSnapshot";
export * from "./archiveSnapshots";
export * from "./collectSongs";
export * from "./dispatchMilestoneSnapshots";
export * from "./dispatchRegularSnapshots";
export * from "./snapshotVideo";
export * from "./getLatestVideos";
export * from "./getVideoInfo";
export * from "./scheduleCleanup";
export * from "./snapshotTick";
export * from "./snapshotVideo";
export * from "./takeBulkSnapshot";

2
packages/crawler/mq/exec/getLatestVideos.ts
View File

@ -1,5 +1,5 @@
import { sql } from "@core/db/dbNew";
import { Job } from "bullmq";
import type { Job } from "bullmq";
import { queueLatestVideos } from "mq/task/queueLatestVideo";
export const getLatestVideosWorker = async (_job: Job): Promise<void> => {

24
packages/crawler/mq/exec/getVideoInfo.ts
View File

@ -1,17 +1,17 @@
import { Job } from "bullmq";
import { getVideoDetails } from "@core/net/getVideoDetails";
import { bilibiliUser, db, videoSnapshot } from "@core/drizzle";
import logger from "@core/log";
import { ClassifyVideoQueue, latestVideosEventsProducer } from "mq/index";
import { getVideoDetails } from "@core/net/getVideoDetails";
import type { Job } from "bullmq";
import {
insertIntoMetadata,
userExistsInBiliUsers,
videoExistsInAllData
videoExistsInAllData,
} from "db/bilibili_metadata";
import { insertIntoSongs } from "mq/task/collectSongs";
import { bilibiliUser, db, videoSnapshot } from "@core/drizzle";
import { eq } from "drizzle-orm";
import { GetVideoInfoJobData } from "mq/schema";
import { snapshotCounter } from "metrics";
import { ClassifyVideoQueue, latestVideosEventsProducer } from "mq/index";
import type { GetVideoInfoJobData } from "mq/schema";
import { insertIntoSongs } from "mq/task/collectSongs";
interface AddSongEventPayload {
eventName: string;
@ -23,7 +23,7 @@ const publishAddsongEvent = async (songID: number, uid: string) =>
latestVideosEventsProducer.publishEvent<AddSongEventPayload>({
eventName: "addSong",
uid: uid,
songID: songID
songID: songID,
});
export const getVideoInfoWorker = async (job: Job<GetVideoInfoJobData>): Promise<void> => {
@ -64,7 +64,7 @@ export const getVideoInfoWorker = async (job: Job<GetVideoInfoJobData>): Promise
title: data.View.title,
publishedAt: new Date(data.View.pubdate * 1000).toISOString(),
duration: data.View.duration,
coverUrl: data.View.pic
coverUrl: data.View.pic,
});
const userExists = await userExistsInBiliUsers(aid);
@ -74,7 +74,7 @@ export const getVideoInfoWorker = async (job: Job<GetVideoInfoJobData>): Promise
username: data.View.owner.name,
desc: data.Card.card.sign,
fans: data.Card.follower,
avatar: data.View.owner.face
avatar: data.View.owner.face,
});
} else {
await db
@ -83,7 +83,7 @@ export const getVideoInfoWorker = async (job: Job<GetVideoInfoJobData>): Promise
username: data.View.owner.name,
desc: data.Card.card.sign,
fans: data.Card.follower,
avatar: data.View.owner.face
avatar: data.View.owner.face,
})
.where(eq(bilibiliUser.uid, uid));
}
@ -98,7 +98,7 @@ export const getVideoInfoWorker = async (job: Job<GetVideoInfoJobData>): Promise
likes: stat.like,
coins: stat.coin,
shares: stat.share,
favorites: stat.favorite
favorites: stat.favorite,
});
snapshotCounter.add(1);

2
packages/crawler/mq/exec/scheduleCleanup.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bullmq";
import logger from "@core/log";
import type { Job } from "bullmq";
import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules";
export const scheduleCleanupWorker = async (_job: Job): Promise<void> => {

20
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -1,21 +1,21 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { getClosetMilestone as closetMilestone } from "@core/lib/milestone";
import logger from "@core/log";
import type { Job } from "bullmq";
import {
bulkGetVideosWithoutProcessingSchedules,
bulkSetSnapshotStatus,
getBulkSnapshotsInNextSecond,
getSnapshotsInNextSecond,
setSnapshotStatus,
videoHasProcessingSchedule
videoHasProcessingSchedule,
} from "db/snapshotSchedule";
import logger from "@core/log";
import { SnapshotQueue } from "mq/index";
import { sql } from "@core/db/dbNew";
import { jobCounter, jobDurationRaw } from "metrics";
import { getClosetMilestone as closetMilestone } from "@core/lib/milestone";
import { SnapshotQueue } from "mq/index";
const priorityMap: { [key: string]: number } = {
milestone: 1,
normal: 3
normal: 3,
};
export const bulkSnapshotTickWorker = async (_job: Job) => {
@ -37,13 +37,13 @@ export const bulkSnapshotTickWorker = async (_job: Job) => {
created_at: schedule.created_at,
started_at: schedule.started_at,
finished_at: schedule.finished_at,
status: schedule.status
status: schedule.status,
};
});
await SnapshotQueue.add(
"bulkSnapshotVideo",
{
schedules: schedulesData
schedules: schedulesData,
},
{ priority: 3 }
);
@ -73,7 +73,7 @@ export const snapshotTickWorker = async (_job: Job) => {
{
aid: Number(aid),
id: Number(schedule.id),
type: schedule.type ?? "normal"
type: schedule.type ?? "normal",
},
{ priority }
);

18
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -1,24 +1,24 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import logger from "@core/log";
import { NetSchedulerError } from "@core/net/delegate";
import type { Job } from "bullmq";
import {
getLatestSnapshot,
scheduleSnapshot,
setSnapshotStatus,
snapshotScheduleExists
snapshotScheduleExists,
} from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata";
import { takeVideoSnapshot } from "mq/task/getVideoStats";
import { getSongsPublihsedAt } from "db/songs";
import { getAdjustedShortTermETA } from "mq/scheduling";
import { NetSchedulerError } from "@core/net/delegate";
import { sql } from "@core/db/dbNew";
import { takeVideoSnapshot } from "mq/task/getVideoStats";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata";
import { closetMilestone } from "./snapshotTick";
const snapshotTypeToTaskMap = {
milestone: "snapshotMilestoneVideo",
normal: "snapshotVideo",
new: "snapshotMilestoneVideo"
new: "snapshotMilestoneVideo",
} as const;
export const snapshotVideoWorker = async (job: Job): Promise<void> => {

22
packages/crawler/mq/exec/takeBulkSnapshot.ts
View File

@ -1,21 +1,21 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import type { SnapshotScheduleType } from "@core/db/schema";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import logger from "@core/log";
import { NetSchedulerError } from "@core/net/delegate";
import type { Job } from "bullmq";
import { updateETA } from "db/eta";
import {
bulkScheduleSnapshot,
bulkSetSnapshotStatus,
getLatestSnapshot,
scheduleSnapshot,
snapshotScheduleExists
snapshotScheduleExists,
} from "db/snapshotSchedule";
import { bulkGetVideoStats } from "net/bulkGetVideoStats";
import logger from "@core/log";
import { NetSchedulerError } from "@core/net/delegate";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval";
import { SnapshotScheduleType } from "@core/db/schema";
import { sql } from "@core/db/dbNew";
import { updateETA } from "db/eta";
import { closetMilestone } from "./snapshotTick";
import { snapshotCounter } from "metrics";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval";
import { bulkGetVideoStats } from "net/bulkGetVideoStats";
import { closetMilestone } from "./snapshotTick";
export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
const schedules: SnapshotScheduleType[] = job.data.schedules;

12
packages/crawler/mq/index.ts
View File

@ -1,22 +1,22 @@
import { Queue, ConnectionOptions, QueueEventsProducer } from "bullmq";
import { type ConnectionOptions, Queue, QueueEventsProducer } from "bullmq";
import { redis } from "bun";
export const LatestVideosQueue = new Queue("latestVideos", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
export const ClassifyVideoQueue = new Queue("classifyVideo", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
export const SnapshotQueue = new Queue("snapshot", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
export const MiscQueue = new Queue("misc", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});
export const latestVideosEventsProducer = new QueueEventsProducer("latestVideos", {
connection: redis as ConnectionOptions
connection: redis as ConnectionOptions,
});

34
packages/crawler/mq/init.ts
View File

@ -1,39 +1,39 @@
import { sql } from "@core/db/dbNew";
import { redis } from "@core/db/redis";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { ClassifyVideoQueue, LatestVideosQueue, MiscQueue, SnapshotQueue } from "mq/index";
import logger from "@core/log";
import { initSnapshotWindowCounts } from "db/snapshotSchedule";
import { redis } from "@core/db/redis";
import { sql } from "@core/db/dbNew";
import { ClassifyVideoQueue, LatestVideosQueue, MiscQueue, SnapshotQueue } from "mq/index";
export async function initMQ() {
await initSnapshotWindowCounts(sql, redis);
await LatestVideosQueue.upsertJobScheduler("getLatestVideos", {
every: 1 * MINUTE,
immediately: true
immediately: true,
});
await ClassifyVideoQueue.upsertJobScheduler("classifyVideos", {
every: 5 * MINUTE,
immediately: true
immediately: true,
});
await LatestVideosQueue.upsertJobScheduler("collectSongs", {
every: 3 * MINUTE,
immediately: true
immediately: true,
});
await SnapshotQueue.upsertJobScheduler(
"snapshotTick",
{
every: 1 * SECOND,
immediately: true
immediately: true,
},
{
opts: {
removeOnComplete: 300,
removeOnFail: 600
}
removeOnFail: 600,
},
}
);
@ -41,39 +41,39 @@ export async function initMQ() {
"bulkSnapshotTick",
{
every: 15 * SECOND,
immediately: true
immediately: true,
},
{
opts: {
removeOnComplete: 60,
removeOnFail: 600
}
removeOnFail: 600,
},
}
);
await SnapshotQueue.upsertJobScheduler("dispatchMilestoneSnapshots", {
every: 5 * MINUTE,
immediately: true
immediately: true,
});
await SnapshotQueue.upsertJobScheduler("dispatchRegularSnapshots", {
every: 30 * MINUTE,
immediately: true
immediately: true,
});
await SnapshotQueue.upsertJobScheduler("dispatchArchiveSnapshots", {
every: 2 * HOUR,
immediately: false
immediately: false,
});
await SnapshotQueue.upsertJobScheduler("scheduleCleanup", {
every: 2 * MINUTE,
immediately: true
immediately: true,
});
await MiscQueue.upsertJobScheduler("collectQueueMetrics", {
every: 3 * SECOND,
immediately: true
immediately: true,
});
logger.log("Message queue initialized.");

6
packages/crawler/mq/scheduling.ts
View File

@ -1,9 +1,9 @@
import type { Psql } from "@core/db/psql.d";
import { HOUR, MINUTE } from "@core/lib";
import { updateETA } from "db/eta";
import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule";
import { truncate } from "utils/truncate";
import { closetMilestone } from "./exec/snapshotTick";
import { HOUR, MINUTE } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
import { updateETA } from "db/eta";
const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);

12
packages/crawler/mq/task/collectSongs.ts
View File

@ -1,11 +1,11 @@
import { sql } from "@core/db/dbNew";
import { aidExistsInSongs, getNotCollectedSongs } from "db/songs";
import logger from "@core/log";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { MINUTE } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
import { db, songs } from "@core/drizzle";
import { and, eq, sql as drizzleSQL } from "drizzle-orm";
import { MINUTE } from "@core/lib";
import logger from "@core/log";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { aidExistsInSongs, getNotCollectedSongs } from "db/songs";
import { and, sql as drizzleSQL, eq } from "drizzle-orm";
export async function collectSongs() {
const aids = await getNotCollectedSongs(sql);
@ -51,7 +51,7 @@ export async function insertIntoSongs(aid: number) {
)
ON CONFLICT DO NOTHING
RETURNING *
`
`;
return data;
}

8
packages/crawler/mq/task/getVideoStats.ts
View File

@ -1,6 +1,6 @@
import { getVideoInfo } from "@core/net/getVideoInfo";
import logger from "@core/log";
import type { Psql } from "@core/db/psql.d";
import logger from "@core/log";
import { getVideoInfo } from "@core/net/getVideoInfo";
import { insertVideoSnapshot } from "db/snapshot";
export interface SnapshotNumber {
@ -52,7 +52,7 @@ export async function takeVideoSnapshot(
shares,
danmakus,
replies,
aid
aid,
});
logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
@ -66,6 +66,6 @@ export async function takeVideoSnapshot(
coins,
shares,
favorites,
time
time,
};
}

12
packages/crawler/mq/task/queueLatestVideo.ts
View File

@ -1,10 +1,10 @@
import { getLatestVideoAids } from "net/getLatestVideoAids";
import { videoExistsInAllData } from "db/bilibili_metadata";
import { sleep } from "utils/sleep";
import type { Psql } from "@core/db/psql.d";
import { SECOND } from "@core/lib";
import logger from "@core/log";
import { videoExistsInAllData } from "db/bilibili_metadata";
import { LatestVideosQueue } from "mq/index";
import type { Psql } from "@core/db/psql.d";
import { getLatestVideoAids } from "net/getLatestVideoAids";
import { sleep } from "utils/sleep";
export async function queueLatestVideos(sql: Psql): Promise<number | null> {
let page = 1;
@ -32,8 +32,8 @@ export async function queueLatestVideos(sql: Psql): Promise<number | null> {
attempts: 100,
backoff: {
type: "fixed",
delay: SECOND * 5
}
delay: SECOND * 5,
},
}
);
videosFound.add(aid);

4
packages/crawler/mq/task/regularSnapshotInterval.ts
View File

@ -1,6 +1,6 @@
import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule";
import { HOUR } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
import { HOUR } from "@core/lib";
import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule";
export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => {
const now = Date.now();

6
packages/crawler/net/bulkGetVideoStats.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate";
import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d";
import networkDelegate from "@core/net/delegate";
/*
* Bulk fetch video metadata from bilibili API
@ -34,6 +34,6 @@ export async function bulkGetVideoStats(aids: number[]): Promise<
}
return {
data: data.data,
time: time
time: time,
};
}

2
packages/crawler/net/getLatestVideoAids.ts
View File

@ -1,5 +1,5 @@
import type { VideoListResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
import type { VideoListResponse } from "@core/net/bilibili.d";
import networkDelegate from "@core/net/delegate";
export async function getLatestVideoAids(

8
packages/crawler/src/bullui.ts
View File

@ -1,8 +1,8 @@
import express from "express";
import { createBullBoard } from "@bull-board/api";
import { BullMQAdapter } from "@bull-board/api/bullMQAdapter.js";
import { ExpressAdapter } from "@bull-board/express";
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue, MiscQueue } from "mq/index";
import express from "express";
import { ClassifyVideoQueue, LatestVideosQueue, MiscQueue, SnapshotQueue } from "mq/index";
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath("/");
@ -12,9 +12,9 @@ createBullBoard({
new BullMQAdapter(LatestVideosQueue),
new BullMQAdapter(ClassifyVideoQueue),
new BullMQAdapter(SnapshotQueue),
new BullMQAdapter(MiscQueue)
new BullMQAdapter(MiscQueue),
],
serverAdapter: serverAdapter
serverAdapter: serverAdapter,
});
const app = express();

6
packages/crawler/src/filterWorker.ts
View File

@ -1,10 +1,10 @@
import { ConnectionOptions, Job, Worker } from "bullmq";
import { redis } from "@core/db/redis";
import logger from "@core/log";
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo";
import { WorkerError } from "mq/schema";
import { lockManager } from "@core/mq/lockManager";
import { type ConnectionOptions, type Job, Worker } from "bullmq";
import Akari from "ml/akari_api";
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo";
import type { WorkerError } from "mq/schema";
const shutdown = async (signal: string, filterWorker: Worker<any, any, string>) => {
logger.log(`${signal} Received: Shutting down workers...`, "mq");

18
packages/crawler/src/worker.ts
View File

@ -1,4 +1,9 @@
import { ConnectionOptions, Job, Worker } from "bullmq";
import { redis } from "@core/db/redis";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { type ConnectionOptions, type Job, Worker } from "bullmq";
import { collectQueueMetrics } from "mq/exec/collectQueueMetrics";
import { directSnapshotWorker } from "mq/exec/directSnapshot";
import {
archiveSnapshotsWorker,
bulkSnapshotTickWorker,
@ -10,14 +15,9 @@ import {
scheduleCleanupWorker,
snapshotTickWorker,
snapshotVideoWorker,
takeBulkSnapshotForVideosWorker
takeBulkSnapshotForVideosWorker,
} from "mq/exec/executors";
import { redis } from "@core/db/redis";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { WorkerError } from "mq/schema";
import { collectQueueMetrics } from "mq/exec/collectQueueMetrics";
import { directSnapshotWorker } from "mq/exec/directSnapshot";
import type { WorkerError } from "mq/schema";
const releaseLockForJob = async (name: string) => {
await lockManager.releaseLock(name);
@ -61,7 +61,7 @@ const latestVideoWorker = new Worker(
connection: redis as ConnectionOptions,
concurrency: 6,
removeOnComplete: { count: 1440 },
removeOnFail: { count: 0 }
removeOnFail: { count: 0 },
}
);

30
packages/palette/src/App.tsx
View File

@ -1,15 +1,15 @@
import "virtual:uno.css";
import { type Oklch } from "culori";
import { Picker } from "./components/Picker/Picker";
import { Switch } from "./Switch";
import { i18nProvider } from "./utils";
import { useTheme } from "./ThemeContext";
import { ColorPalette } from "./components/Palette";
import { Buttons, Paragraph, SearchBar } from "./components/Components";
import { AnimatePresence, motion } from "motion/react";
import { Moon, Sun } from "lucide-react";
import type { Oklch } from "culori";
import { useAtom } from "jotai";
import { atomWithStorage } from "jotai/utils";
import { Moon, Sun } from "lucide-react";
import { AnimatePresence, motion } from "motion/react";
import { Buttons, Paragraph, SearchBar } from "./components/Components";
import { ColorPalette } from "./components/Palette";
import { Picker } from "./components/Picker/Picker";
import { Switch } from "./Switch";
import { useTheme } from "./ThemeContext";
import { i18nProvider } from "./utils";
const defaultColor: Oklch = { mode: "oklch", h: 29.2339, c: 0.244572, l: 0.596005 };
@ -60,16 +60,22 @@ function App() {
return (
<div className="min-h-screen my-12 sm:px-6">
<div className="max-w-7xl mx-auto">
<h1 className="text-3xl font-bold mb-8 ml-3 text-on-background">CVSA Color Palette Generator</h1>
<h1 className="text-3xl font-bold mb-8 ml-3 text-on-background">
CVSA Color Palette Generator
</h1>
<div className="grid grid-cols-1 md:grid-cols-2 lg:[grid-template-columns:2fr_3fr] xl:grid-cols-3 gap-8">
{/* Left Column - Color Picker */}
<div className="xl:col-span-1 sm:bg-white sm:dark:bg-zinc-800 rounded-lg shadow-sm p-3 sm:p-6">
<h2 className="text-xl font-semibold text-on-background mb-4">Color Selection</h2>
<h2 className="text-xl font-semibold text-on-background mb-4">
Color Selection
</h2>
<div className="space-y-6">
<div>
<label className="block font-bold text-on-background mb-2">OKLCH Color Picker</label>
<label className="block font-bold text-on-background mb-2">
OKLCH Color Picker
</label>
<div className="mx-3">
<Picker
className="m-3"

16
packages/palette/src/Switch.tsx
View File

@ -6,7 +6,13 @@ interface SwitchProps {
label?: string;
}
export const Switch: React.FC<SwitchProps> = ({ checked, onChange, disabled = false, className = "", label }) => {
export const Switch: React.FC<SwitchProps> = ({
checked,
onChange,
disabled = false,
className = "",
label,
}) => {
const handleToggle = () => {
if (!disabled) {
onChange(!checked);
@ -22,7 +28,9 @@ export const Switch: React.FC<SwitchProps> = ({ checked, onChange, disabled = fa
disabled={disabled}
className={`relative flex items-center justify-center w-12 h-6 rounded-full transition-all duration-300
focus:outline-none focus:ring-2 focus:ring-green-500 focus:ring-offset-2 ${
disabled ? "cursor-not-allowed opacity-50" : "cursor-pointer hover:scale-105"
disabled
? "cursor-not-allowed opacity-50"
: "cursor-pointer hover:scale-105"
} ${checked ? "bg-green-500" : "bg-zinc-300 dark:bg-zinc-600"}`}
aria-checked={checked}
aria-disabled={disabled}
@ -40,7 +48,9 @@ export const Switch: React.FC<SwitchProps> = ({ checked, onChange, disabled = fa
{label && (
<label
className={`text-sm font-medium ${
disabled ? "text-gray-400 dark:text-gray-500" : "text-gray-800 dark:text-gray-200"
disabled
? "text-gray-400 dark:text-gray-500"
: "text-gray-800 dark:text-gray-200"
}`}
onClick={handleToggle}
>

2
packages/palette/src/ThemeContext.tsx
View File

@ -1,6 +1,6 @@
// ThemeContext.tsx
import { createContext, useCallback, useContext, useEffect, useState } from "react";
import { type ThemeMode } from "./colorTokens";
import type { ThemeMode } from "./colorTokens";
type ThemeContextType = {
theme: ThemeMode;

Some files were not shown because too many files have changed in this diff Show More