alikia2x/cvsa
1
0
Fork 0
Code

ref: cleanup code

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-10-06 02:10:46 +08:00
parent 1d0a165723
commit 02a2a845da
86 changed files with 398 additions and 435 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bun.lock
View File

@ -41,10 +41,12 @@
"version": "0.0.10",
"dependencies": {
"@koshnic/ratelimit": "^1.0.3",
"@types/luxon": "^3.7.1",
"chalk": "^5.4.1",
"drizzle-orm": "^0.44.4",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"luxon": "^3.7.2",
"postgres": "^3.4.5",
"winston": "^3.17.0",
},

2
packages/backend/db/latestSnapshots.ts
View File

@ -1,5 +1,5 @@
import { sql } from "@core/db/dbNew";
import type { LatestSnapshotType } from "@core/db/schema.d.ts";
import type { LatestSnapshotType } from "@core/db/schema";
export async function getVideosInViewsRange(minViews: number, maxViews: number) {
return sql<LatestSnapshotType[]>`

2
packages/backend/db/snapshots.ts
View File

@ -1,5 +1,5 @@
import { sql } from "@core/db/dbNew";
import type { VideoSnapshotType } from "@core/db/schema.d.ts";
import type { VideoSnapshotType } from "@core/db/schema";
export async function getVideoSnapshots(
aid: number,

6
packages/backend/lib/auth/captchaDifficulty.ts
View File

@ -1,7 +1,7 @@
import { Psql } from "@core/db/psql";
import { SlidingWindow } from "@core/mq/slidingWindow.ts";
import { redis } from "@core/db/redis.ts";
import { getIdentifier } from "@/middleware/rateLimiters.ts";
import { SlidingWindow } from "@core/mq/slidingWindow";
import { redis } from "@core/db/redis";
import { getIdentifier } from "@/middleware/rateLimiters";
import { Context } from "hono";
type seconds = number;

4
packages/backend/middleware/bodyLimits.ts
View File

@ -1,10 +1,10 @@
import { bodyLimit } from "hono/body-limit";
import { ErrorResponse } from "../src/schema";
import { ErrorResponse } from "@/src/schema";
export const bodyLimitForPing = bodyLimit({
maxSize: 14000,
onError: (c) => {
const res: ErrorResponse<string> = {
const res: ErrorResponse = {
message: "Body too large",
errors: ["Body should not be larger than 14kB."],
code: "BODY_TOO_LARGE"

14
packages/backend/middleware/captcha.ts
View File

@ -1,15 +1,15 @@
import { Context, Next } from "hono";
import { ErrorResponse } from "src/schema";
import { SlidingWindow } from "@core/mq/slidingWindow.ts";
import { getCaptchaConfigMaxDuration, getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts";
import { sqlCred } from "@core/db/dbNew.ts";
import { redis } from "@core/db/redis.ts";
import { SlidingWindow } from "@core/mq/slidingWindow";
import { getCaptchaConfigMaxDuration, getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty";
import { sqlCred } from "@core/db/dbNew";
import { redis } from "@core/db/redis";
import { verify } from "hono/jwt";
import { JwtTokenInvalid, JwtTokenExpired } from "hono/utils/jwt/types";
import { getJWTsecret } from "@/lib/auth/getJWTsecret.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import { getJWTsecret } from "@/lib/auth/getJWTsecret";
import { lockManager } from "@core/mq/lockManager";
import { object, string, number, ValidationError } from "yup";
import { getIdentifier } from "@/middleware/rateLimiters.ts";
import { getIdentifier } from "@/middleware/rateLimiters";
const tokenSchema = object({
exp: number().integer(),

4
packages/backend/middleware/rateLimiters.ts
View File

@ -1,10 +1,10 @@
import type { BlankEnv } from "hono/types";
import { getConnInfo } from "hono/bun";
import { Context, Next } from "hono";
import { generateRandomId } from "@core/lib/randomID.ts";
import { generateRandomId } from "@core/lib/randomID";
import { RateLimiter } from "@koshnic/ratelimit";
import { ErrorResponse } from "@/src/schema";
import { redis } from "@core/db/redis.ts";
import { redis } from "@core/db/redis";
export const getUserIP = (c: Context) => {
let ipAddr = null;

6
packages/backend/routes/captcha/[id]/result/GET.ts
View File

@ -1,10 +1,10 @@
import { Context } from "hono";
import { Bindings, BlankEnv } from "hono/types";
import { ErrorResponse } from "src/schema";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import { sign } from "hono/jwt";
import { generateRandomId } from "@core/lib/randomID.ts";
import { getJWTsecret } from "lib/auth/getJWTsecret.ts";
import { generateRandomId } from "@core/lib/randomID";
import { getJWTsecret } from "lib/auth/getJWTsecret";
interface CaptchaResponse {
success: boolean;

6
packages/backend/routes/captcha/difficulty/GET.ts
View File

@ -1,8 +1,8 @@
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import { object, string, ValidationError } from "yup";
import { ErrorResponse } from "src/schema";
import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts";
import { sqlCred } from "@core/db/dbNew.ts";
import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty";
import { sqlCred } from "@core/db/dbNew";
const queryParamsSchema = object({
route: string().matches(/(?:GET|POST|PUT|PATCH|DELETE)-\/.*/g)

6
packages/backend/routes/captcha/session/POST.ts
View File

@ -1,6 +1,6 @@
import { createHandlers } from "src/utils.ts";
import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts";
import { sqlCred } from "@core/db/dbNew.ts";
import { createHandlers } from "src/utils";
import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty";
import { sqlCred } from "@core/db/dbNew";
import { object, string, ValidationError } from "yup";
import { CaptchaSessionResponse, ErrorResponse } from "@/src/schema";
import type { ContentfulStatusCode } from "hono/utils/http-status";

6
packages/backend/routes/index.ts
View File

@ -1,6 +1,6 @@
import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "lib/const/singers.ts";
import { VERSION } from "src/main.ts";
import { createHandlers } from "src/utils.ts";
import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "lib/const/singers";
import { VERSION } from "src/main";
import { createHandlers } from "src/utils";
export const rootHandler = createHandlers((c) => {
let singer: Singer | Singer[];

22
packages/backend/routes/login/session/POST.ts
View File

@ -1,7 +1,7 @@
import { Context } from "hono";
import { Bindings, BlankEnv } from "hono/types";
import { ErrorResponse, LoginResponse } from "src/schema";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import { sqlCred } from "@core/db/dbNew";
import { object, string, ValidationError } from "yup";
import { setCookie } from "hono/cookie";
@ -27,12 +27,12 @@ export const loginHandler = createHandlers(
`;
if (result.length === 0) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: `User does not exist.`,
errors: [`User ${username} does not exist.`],
code: "ENTITY_NOT_FOUND"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
}
const storedPassword = result[0].password;
@ -43,7 +43,7 @@ export const loginHandler = createHandlers(
const passwordAreSame = await Argon2id.verify(storedPassword, submittedPassword);
if (!passwordAreSame) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Incorrect password.",
errors: [],
i18n: {
@ -51,7 +51,7 @@ export const loginHandler = createHandlers(
},
code: "INVALID_CREDENTIALS"
};
return c.json<ErrorResponse<string>>(response, 401);
return c.json<ErrorResponse>(response, 401);
}
const sessionID = await createLoginSession(uid, c);
@ -79,26 +79,26 @@ export const loginHandler = createHandlers(
return c.json<LoginResponse>(response, 200);
} catch (e) {
if (e instanceof ValidationError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid registration data.",
errors: e.errors,
code: "INVALID_PAYLOAD"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else if (e instanceof SyntaxError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid JSON payload.",
errors: [e.message],
code: "INVALID_FORMAT"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Unknown error.",
errors: [(e as Error).message],
code: "UNKNOWN_ERROR"
};
return c.json<ErrorResponse<string>>(response, 500);
return c.json<ErrorResponse>(response, 500);
}
}
}

6
packages/backend/routes/ping/index.ts
View File

@ -1,6 +1,6 @@
import { getClientIP } from "middleware/logger.ts";
import { createHandlers } from "src/utils.ts";
import { VERSION } from "src/main.ts";
import { getClientIP } from "middleware/logger";
import { createHandlers } from "src/utils";
import { VERSION } from "src/main";
export const pingHandler = createHandlers(async (c) => {
const requestHeaders = c.req.raw.headers;

20
packages/backend/routes/session/[id]/DELETE.ts
View File

@ -1,9 +1,9 @@
import { Context } from "hono";
import { Bindings, BlankEnv } from "hono/types";
import { ErrorResponse } from "src/schema";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import { sqlCred } from "@core/db/dbNew";
import { object, string, ValidationError } from "yup";
import { ValidationError } from "yup";
import { setCookie } from "hono/cookie";
const loginSessionExists = async (sessionID: string) => {
@ -22,12 +22,12 @@ export const logoutHandler = createHandlers(async (c: Context<BlankEnv & { Bindi
const exists = loginSessionExists(session_id);
if (!exists) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Cannot found given session_id.",
errors: [`Session ${session_id} not found`],
code: "ENTITY_NOT_FOUND"
};
return c.json<ErrorResponse<string>>(response, 404);
return c.json<ErrorResponse>(response, 404);
}
await sqlCred`
@ -50,26 +50,26 @@ export const logoutHandler = createHandlers(async (c: Context<BlankEnv & { Bindi
return c.body(null, 204);
} catch (e) {
if (e instanceof ValidationError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid registration data.",
errors: e.errors,
code: "INVALID_PAYLOAD"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else if (e instanceof SyntaxError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid JSON payload.",
errors: [e.message],
code: "INVALID_FORMAT"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Unknown error.",
errors: [(e as Error).message],
code: "UNKNOWN_ERROR"
};
return c.json<ErrorResponse<string>>(response, 500);
return c.json<ErrorResponse>(response, 500);
}
}
});

20
packages/backend/routes/user/POST.ts
View File

@ -1,9 +1,9 @@
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import Argon2id from "@rabbit-company/argon2id";
import { object, string, ValidationError } from "yup";
import type { Context } from "hono";
import type { Bindings, BlankEnv, BlankInput } from "hono/types";
import { sqlCred } from "@core/db/dbNew.ts";
import { sqlCred } from "@core/db/dbNew";
import { ErrorResponse, SignUpResponse } from "src/schema";
import { generateRandomId } from "@core/lib/randomID";
import { getUserIP } from "@/middleware/rateLimiters";
@ -79,7 +79,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => {
const uid = await getUserIDByName(username);
if (!uid) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Cannot find registered user.",
errors: [`Cannot find user ${username} in table 'users'.`],
code: "ENTITY_NOT_FOUND",
@ -90,7 +90,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => {
}
}
};
return c.json<ErrorResponse<string>>(response, 500);
return c.json<ErrorResponse>(response, 500);
}
const sessionID = await createLoginSession(uid, c);
@ -115,26 +115,26 @@ export const registerHandler = createHandlers(async (c: ContextType) => {
return c.json<SignUpResponse>(response, 201);
} catch (e) {
if (e instanceof ValidationError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid registration data.",
errors: e.errors,
code: "INVALID_PAYLOAD"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else if (e instanceof SyntaxError) {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Invalid JSON payload.",
errors: [e.message],
code: "INVALID_FORMAT"
};
return c.json<ErrorResponse<string>>(response, 400);
return c.json<ErrorResponse>(response, 400);
} else {
const response: ErrorResponse<string> = {
const response: ErrorResponse = {
message: "Unknown error.",
errors: [(e as Error).message],
code: "UNKNOWN_ERROR"
};
return c.json<ErrorResponse<string>>(response, 500);
return c.json<ErrorResponse>(response, 500);
}
}
});

2
packages/backend/routes/user/session/[id]/GET.ts
View File

@ -1,7 +1,7 @@
import { Context } from "hono";
import { Bindings, BlankEnv } from "hono/types";
import { ErrorResponse } from "src/schema";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import { sqlCred } from "@core/db/dbNew";
import { UserType } from "@core/db/schema";

16
packages/backend/routes/video/[id]/info.ts
View File

@ -1,14 +1,14 @@
import logger from "@core/log/logger.ts";
import { redis } from "@core/db/redis.ts";
import { sql } from "@core/db/dbNew.ts";
import logger from "@core/log";
import { redis } from "@core/db/redis";
import { sql } from "@core/db/dbNew";
import { number, ValidationError } from "yup";
import { createHandlers } from "@/src/utils.ts";
import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo.ts";
import { idSchema } from "./snapshots.ts";
import { NetSchedulerError } from "@core/net/delegate.ts";
import { createHandlers } from "@/src/utils";
import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo";
import { idSchema } from "./snapshots";
import { NetSchedulerError } from "@core/net/delegate";
import type { Context } from "hono";
import type { BlankEnv, BlankInput } from "hono/types";
import type { VideoInfoData } from "@core/net/bilibili.d.ts";
import type { VideoInfoData } from "@core/net/bilibili";
import { startTime, endTime } from "hono/timing";
const CACHE_EXPIRATION_SECONDS = 60;

6
packages/backend/routes/video/[id]/snapshots.ts
View File

@ -1,8 +1,8 @@
import type { Context } from "hono";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import type { BlankEnv, BlankInput } from "hono/types";
import { getVideoSnapshots, getVideoSnapshotsByBV } from "db/snapshots.ts";
import type { VideoSnapshotType } from "@core/db/schema.d.ts";
import { getVideoSnapshots, getVideoSnapshotsByBV } from "db/snapshots";
import type { VideoSnapshotType } from "@core/db/schema";
import { boolean, mixed, number, object, ValidationError } from "yup";
import { ErrorResponse } from "src/schema";
import { startTime, endTime } from "hono/timing";

2
packages/backend/routes/videos/GET.ts
View File

@ -1,5 +1,5 @@
import type { Context } from "hono";
import { createHandlers } from "src/utils.ts";
import { createHandlers } from "src/utils";
import type { BlankEnv, BlankInput } from "hono/types";
import { number, object, ValidationError } from "yup";
import { ErrorResponse } from "src/schema";

8
packages/backend/src/main.ts
View File

@ -1,9 +1,9 @@
import { Hono } from "hono";
import type { TimingVariables } from "hono/timing";
import { startServer } from "./startServer.ts";
import { configureRoutes } from "./routing.ts";
import { configureMiddleWares } from "./middleware.ts";
import { notFoundRoute } from "routes/404.ts";
import { startServer } from "./startServer";
import { configureRoutes } from "./routing";
import { configureMiddleWares } from "./middleware";
import { notFoundRoute } from "routes/404";
type Variables = TimingVariables;
const app = new Hono<{ Variables: Variables }>();

12
packages/backend/src/middleware.ts
View File

@ -2,13 +2,13 @@ import { Hono } from "hono";
import { timing } from "hono/timing";
import { Variables } from "hono/types";
import { pingHandler } from "routes/ping";
import { logger } from "middleware/logger.ts";
import { logger } from "middleware/logger";
import { corsMiddleware } from "@/middleware/cors";
import { contentType } from "middleware/contentType.ts";
import { captchaMiddleware } from "middleware/captcha.ts";
import { bodyLimitForPing } from "middleware/bodyLimits.ts";
import { registerRateLimiter } from "middleware/rateLimiters.ts";
import { preetifyResponse } from "middleware/preetifyResponse.ts";
import { contentType } from "middleware/contentType";
import { captchaMiddleware } from "middleware/captcha";
import { bodyLimitForPing } from "middleware/bodyLimits";
import { registerRateLimiter } from "middleware/rateLimiters";
import { preetifyResponse } from "middleware/preetifyResponse";
export function configureMiddleWares(app: Hono<{ Variables: Variables }>) {
app.use("*", corsMiddleware);

2
packages/backend/src/routing.ts
View File

@ -5,7 +5,7 @@ import { videoInfoHandler, getSnapshotsHanlder } from "routes/video";
import { Hono } from "hono";
import { Variables } from "hono/types";
import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha";
import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET.ts";
import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET";
import { getVideosHanlder } from "@/routes/videos";
import { loginHandler } from "@/routes/login/session/POST";
import { logoutHandler } from "@/routes/session";

2
packages/backend/src/schema.d.ts vendored
View File

@ -15,7 +15,7 @@ export type ErrorCode =
export interface ErrorResponse<E = string> {
code: ErrorCode;
message: string;
errors: E[] = [];
errors: E[];
i18n?: {
key: string;
values?: {

1
packages/core/db/index.ts Normal file
View File

@ -0,0 +1 @@
export * from "./snapshots"

30
packages/core/db/snapshots/getClosetSnapshot.ts Normal file
View File

@ -0,0 +1,30 @@
import { dbMain } from "@core/drizzle";
import { sql } from "drizzle-orm";
export const getClosestSnapshot = async (aid: number, targetTime: Date) => {
const closest = await dbMain.execute<{ created_at: Date; views: number }>(sql`
SELECT created_at, views
FROM (
(SELECT created_at, views, 'later' AS type
FROM video_snapshot
WHERE aid = ${aid}
AND created_at >= ${targetTime.toISOString()}
ORDER BY created_at
LIMIT 1)
UNION ALL
(SELECT created_at, views, 'earlier' AS type
FROM video_snapshot
WHERE aid = ${aid}
AND created_at <= ${targetTime.toISOString()}
ORDER BY created_at DESC
LIMIT 1)
) AS combined
ORDER BY
CASE
WHEN created_at >= ${targetTime.toISOString()} THEN created_at -${targetTime.toISOString()}
ELSE ${targetTime.toISOString()} - created_at
END
LIMIT 1;
`);
return closest[0] || null;
};

11
packages/core/db/snapshots/getLatestSnapshot.ts Normal file
View File

@ -0,0 +1,11 @@
import { dbMain } from "@core/drizzle";
import { latestVideoSnapshot } from "@core/drizzle/main/schema";
import { eq } from "drizzle-orm";
export const getLatestSnapshot = async (aid: number) =>{
const result = await dbMain.select().from(latestVideoSnapshot).where(eq(latestVideoSnapshot.aid, aid)).limit(1);
if (result.length === 0) {
return null;
}
return result[0];
}

2
packages/core/db/snapshots/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from "./getLatestSnapshot";
export * from "./getClosetSnapshot";

30
packages/core/db/snapshots/milestone.ts Normal file
View File

@ -0,0 +1,30 @@
import { MINUTE, HOUR, getClosetMilestone, getMileStoneETAfactor, truncate } from "@core/lib";
import { getLatestSnapshot, getClosestSnapshot } from "@core/db";
export const getShortTermETA = async (aid: number) => {
const DELTA = 1e-5;
let minETAHours = Infinity;
const timeIntervals = [20 * MINUTE, HOUR, 3 * HOUR, 6 * HOUR, 24 * HOUR, 72 * HOUR, 168 * HOUR];
const currentTimestamp = new Date().getTime();
const latestSnapshot = await getLatestSnapshot(aid);
for (const timeInterval of timeIntervals) {
const date = new Date(currentTimestamp - timeInterval);
const snapshot = await getClosestSnapshot(aid, date);
if (!snapshot) continue;
const latestSnapshotTime = new Date(latestSnapshot.time).getTime();
const currentSnapshotTime = new Date(snapshot.created_at).getTime();
const hoursDiff = (latestSnapshotTime - currentSnapshotTime) / HOUR;
const viewsDiff = latestSnapshot.views - snapshot.views;
if (viewsDiff <= 0) continue;
const speed = viewsDiff / (hoursDiff + DELTA);
const target = getClosetMilestone(latestSnapshot.views);
const viewsToIncrease = target - latestSnapshot.views;
const eta = viewsToIncrease / (speed + DELTA);
let factor = getMileStoneETAfactor(viewsToIncrease);
factor = truncate(factor, 4.5, 100);
const adjustedETA = eta / factor;
if (adjustedETA < minETAHours) {
minETAHours = adjustedETA;
}
}
};

4
packages/core/lib/index.ts Normal file
View File

@ -0,0 +1,4 @@
export * from "./math";
export * from "./milestone";
export * from "./randomID";
export * from "./time";

3
packages/core/lib/math.ts Normal file
View File

@ -0,0 +1,3 @@
export const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
export const truncate = (num: number, min: number, max: number) => Math.max(min, Math.min(num, max));

20
packages/core/lib/milestone.ts Normal file
View File

@ -0,0 +1,20 @@
import { log } from "@core/lib";
export const getMileStoneETAfactor = (x: number) => {
const a = 1.054;
const b = 4.5;
const c = 100;
const u = 0.601;
const g = 455;
if (x > g) {
return log(b / log(x + 1), a);
} else {
return log(b / log(x + c), a) + u;
}
};
export const getClosetMilestone = (views: number) => {
if (views < 100000) return 100000;
if (views < 1000000) return 1000000;
return Math.ceil(views / 1000000) * 1000000;
};

0
packages/core/const/time.ts → packages/core/lib/time.ts
View File

0
packages/core/log/logger.ts → packages/core/log/index.ts
View File

12
packages/core/log/test.ts
View File

@ -1,12 +0,0 @@
import logger from "@core/log/logger.ts";
logger.error(Error("test error"), "test service");
logger.debug(`some string`);
logger.debug(`hello`, "servicename");
logger.debug(`hello`, "servicename", "codepath.ts");
logger.log("something");
logger.log("foo", "service");
logger.log("foo", "db", "insert.ts");
logger.warn("warn");
logger.error("error");
logger.verbose("error");

2
packages/core/mq/lockManager.ts
View File

@ -1,5 +1,5 @@
import { Redis } from "ioredis";
import { redis } from "@core/db/redis.ts";
import { redis } from "@core/db/redis";
class LockManager {
private redis: Redis;

2
packages/core/mq/multipleRateLimiter.ts
View File

@ -1,5 +1,5 @@
import { RateLimiter as Limiter } from "@koshnic/ratelimit";
import { redis } from "@core/db/redis.ts";
import { redis } from "@core/db/redis";
export interface RateLimiterConfig {
duration: number;

6
packages/core/net/delegate.ts
View File

@ -1,7 +1,7 @@
import logger from "@core/log/logger.ts";
import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter.ts";
import logger from "@core/log";
import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter";
import { ReplyError } from "ioredis";
import { SECOND } from "@core/const/time.ts";
import { SECOND } from "@core/lib";
import { spawn, SpawnOptions } from "child_process";
export function spawnPromise(

6
packages/core/net/getVideoInfo.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate.ts";
import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d.ts";
import logger from "@core/log/logger.ts";
import networkDelegate from "@core/net/delegate";
import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
/*
* Fetch video metadata from bilibili API

4
packages/core/package.json
View File

@ -8,11 +8,13 @@
},
"dependencies": {
"@koshnic/ratelimit": "^1.0.3",
"@types/luxon": "^3.7.1",
"chalk": "^5.4.1",
"drizzle-orm": "^0.44.4",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"luxon": "^3.7.2",
"postgres": "^3.4.5",
"drizzle-orm": "^0.44.4",
"winston": "^3.17.0"
},
"devDependencies": {

18
packages/core/test/lib/randomID.test.ts
View File

@ -1,18 +0,0 @@
import { describe, expect, it } from "vitest";
import { generateRandomId } from "@core/lib/randomID.ts";
describe("generateRandomId", () => {
it("should generate an ID of the specified length", () => {
const length = 15;
const id = generateRandomId(length);
expect(id).toHaveLength(length);
});
it("should generate an ID containing only allowed characters", () => {
const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789";
const id = generateRandomId(20);
for (const char of id) {
expect(allowedChars).toContain(char);
}
});
});

16
packages/crawler/db/bilibili_metadata.ts
View File

@ -1,5 +1,5 @@
import type { Psql } from "@core/db/psql.d.ts";
import { BiliVideoMetadataType, BiliUserType } from "@core/db/schema";
import type { Psql } from "@core/db/psql.d";
import { BiliUserType, BiliVideoMetadataType } from "@core/db/schema";
import { AkariModelVersion } from "ml/const";
export async function videoExistsInAllData(sql: Psql, aid: number) {
@ -30,7 +30,7 @@ export async function insertVideoLabel(sql: Psql, aid: number, label: number) {
}
export async function getVideoInfoFromAllData(sql: Psql, aid: number) {
const rows = await sql<AllDataType[]>`
const rows = await sql<BiliVideoMetadataType[]>`
SELECT * FROM bilibili_metadata WHERE aid = ${aid}
`;
const row = rows[0];
@ -52,16 +52,6 @@ export async function getVideoInfoFromAllData(sql: Psql, aid: number) {
};
}
export async function getUnArchivedBiliUsers(sql: Psql) {
const rows = await sql<{ uid: number }[]>`
SELECT ad.uid
FROM bilibili_metadata ad
LEFT JOIN bilibili_user bu ON ad.uid = bu.uid
WHERE bu.uid IS NULL;
`;
return rows.map((row) => row.uid);
}
export async function setBiliVideoStatus(sql: Psql, aid: number, status: number) {
await sql`
UPDATE bilibili_metadata SET status = ${status} WHERE aid = ${aid}

4
packages/crawler/db/snapshot.ts
View File

@ -1,6 +1,6 @@
import { LatestSnapshotType } from "@core/db/schema";
import { SnapshotNumber } from "mq/task/getVideoStats.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { SnapshotNumber } from "mq/task/getVideoStats";
import type { Psql } from "@core/db/psql.d";
export async function getVideosNearMilestone(sql: Psql) {
const queryResult = await sql<LatestSnapshotType[]>`

12
packages/crawler/db/snapshotSchedule.ts
View File

@ -1,10 +1,10 @@
import type { SnapshotScheduleType } from "@core/db/schema.d.ts";
import logger from "@core/log/logger.ts";
import { MINUTE } from "@core/const/time.ts";
import { redis } from "@core/db/redis.ts";
import type { SnapshotScheduleType } from "@core/db/schema.d";
import logger from "@core/log";
import { MINUTE } from "@core/lib";
import { redis } from "@core/db/redis";
import { Redis } from "ioredis";
import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre";
import type { Psql } from "@core/db/psql.d";
const REDIS_KEY = "cvsa:snapshot_window_counts";

4
packages/crawler/db/songs.ts
View File

@ -1,5 +1,5 @@
import type { Psql } from "@core/db/psql.d.ts";
import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts";
import type { Psql } from "@core/db/psql.d";
import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre";
export async function getNotCollectedSongs(sql: Psql) {
const rows = await sql<{ aid: number }[]>`

7
packages/crawler/main.ts
View File

@ -1,7 +0,0 @@
// DENO ASK ME TO EXPORT SOMETHING WHEN 'name' IS SPECIFIED
// AND IF I DON'T SPECIFY 'name', THE --filter FLAG IN `deno task` WON'T WORK.
// I DONT'T KNOW WHY
// SO HERE'S A PLACHOLDER EXPORT FOR DENO:
export const DENO = "FUCK YOU DENO";
// Oh, maybe export the version is a good idea
export const VERSION = "1.0.26";

10
packages/crawler/ml/akari.ts
View File

@ -1,7 +1,7 @@
import { AIManager } from "ml/manager.ts";
import { AIManager } from "ml/manager";
import * as ort from "onnxruntime-node";
import logger from "@core/log/logger.ts";
import { WorkerError } from "mq/schema.ts";
import logger from "@core/log";
import { WorkerError } from "mq/schema";
import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers";
import { AkariModelVersion } from "./const";
@ -91,10 +91,6 @@ class AkariProto extends AIManager {
}
return probabilities.indexOf(Math.max(...probabilities));
}
public getModelVersion(): string {
return this.modelVersion;
}
}
const Akari = new AkariProto();

4
packages/crawler/ml/manager.ts
View File

@ -1,6 +1,6 @@
import * as ort from "onnxruntime-node";
import logger from "@core/log/logger.ts";
import { WorkerError } from "mq/schema.ts";
import logger from "@core/log";
import { WorkerError } from "mq/schema";
export class AIManager {
public sessions: { [key: string]: ort.InferenceSession } = {};

10
packages/crawler/mq/exec/archiveSnapshots.ts
View File

@ -1,9 +1,9 @@
import { Job } from "bullmq";
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import { getLatestVideoSnapshot } from "db/snapshot.ts";
import { MINUTE } from "@core/const/time.ts";
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { getLatestVideoSnapshot } from "db/snapshot";
import { MINUTE } from "@core/lib";
import { sql } from "@core/db/dbNew";
function getNextSaturdayMidnightTimestamp(): number {

20
packages/crawler/mq/exec/classifyVideo.ts
View File

@ -1,14 +1,14 @@
import { Job } from "bullmq";
import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "../../db/bilibili_metadata.ts";
import Akari from "ml/akari.ts";
import { ClassifyVideoQueue } from "mq/index.ts";
import logger from "@core/log/logger.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import { aidExistsInSongs } from "db/songs.ts";
import { insertIntoSongs } from "mq/task/collectSongs.ts";
import { scheduleSnapshot } from "db/snapshotSchedule.ts";
import { MINUTE } from "@core/const/time.ts";
import { sql } from "@core/db/dbNew.ts";
import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "../../db/bilibili_metadata";
import Akari from "ml/akari";
import { ClassifyVideoQueue } from "mq/index";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { aidExistsInSongs } from "db/songs";
import { insertIntoSongs } from "mq/task/collectSongs";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { MINUTE } from "@core/lib";
import { sql } from "@core/db/dbNew";
export const classifyVideoWorker = async (job: Job) => {
const aid = job.data.aid;

2
packages/crawler/mq/exec/collectSongs.ts
View File

@ -1,5 +1,5 @@
import { Job } from "bullmq";
import { collectSongs } from "mq/task/collectSongs.ts";
import { collectSongs } from "mq/task/collectSongs";
export const collectSongsWorker = async (_job: Job): Promise<void> => {
await collectSongs();

12
packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts
View File

@ -1,10 +1,10 @@
import { Job } from "bullmq";
import { getVideosNearMilestone } from "db/snapshot.ts";
import { getAdjustedShortTermETA } from "mq/scheduling.ts";
import { truncate } from "utils/truncate.ts";
import { scheduleSnapshot } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { getVideosNearMilestone } from "db/snapshot";
import { getAdjustedShortTermETA } from "mq/scheduling";
import { truncate } from "utils/truncate";
import { scheduleSnapshot } from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { sql } from "@core/db/dbNew";
export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {

16
packages/crawler/mq/exec/dispatchRegularSnapshots.ts
View File

@ -1,12 +1,12 @@
import { Job } from "bullmq";
import { getLatestVideoSnapshot } from "db/snapshot.ts";
import { truncate } from "utils/truncate.ts";
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { HOUR, MINUTE, WEEK } from "@core/const/time.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts";
import { sql } from "@core/db/dbNew.ts";
import { getLatestVideoSnapshot } from "db/snapshot";
import { truncate } from "utils/truncate";
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, WEEK } from "@core/lib";
import { lockManager } from "@core/mq/lockManager";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval";
import { sql } from "@core/db/dbNew";
export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise<void> => {
try {

20
packages/crawler/mq/exec/executors.ts
View File

@ -1,10 +1,10 @@
export * from "mq/exec/getLatestVideos.ts";
export * from "./getVideoInfo.ts";
export * from "./collectSongs.ts";
export * from "./takeBulkSnapshot.ts";
export * from "./archiveSnapshots.ts";
export * from "./dispatchMilestoneSnapshots.ts";
export * from "./dispatchRegularSnapshots.ts";
export * from "./snapshotVideo.ts";
export * from "./scheduleCleanup.ts";
export * from "./snapshotTick.ts";
export * from "./getLatestVideos";
export * from "./getVideoInfo";
export * from "./collectSongs";
export * from "./takeBulkSnapshot";
export * from "./archiveSnapshots";
export * from "./dispatchMilestoneSnapshots";
export * from "./dispatchRegularSnapshots";
export * from "./snapshotVideo";
export * from "./scheduleCleanup";
export * from "./snapshotTick";

2
packages/crawler/mq/exec/getLatestVideos.ts
View File

@ -1,6 +1,6 @@
import { sql } from "@core/db/dbNew";
import { Job } from "bullmq";
import { queueLatestVideos } from "mq/task/queueLatestVideo.ts";
import { queueLatestVideos } from "mq/task/queueLatestVideo";
export const getLatestVideosWorker = async (_job: Job): Promise<void> => {
await queueLatestVideos(sql);

4
packages/crawler/mq/exec/getVideoInfo.ts
View File

@ -1,6 +1,6 @@
import { Job } from "bullmq";
import { insertVideoInfo } from "mq/task/getVideoDetails.ts";
import logger from "@core/log/logger.ts";
import { insertVideoInfo } from "mq/task/getVideoDetails";
import logger from "@core/log";
import { sql } from "@core/db/dbNew";
export const getVideoInfoWorker = async (job: Job): Promise<void> => {

10
packages/crawler/mq/exec/scheduleCleanup.ts
View File

@ -1,10 +1,10 @@
import { Job } from "bullmq";
import { sql } from "@core/db/dbNew";
import logger from "@core/log/logger.ts";
import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule.ts";
import { SECOND } from "@core/const/time.ts";
import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount.ts";
import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules.ts";
import logger from "@core/log";
import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule";
import { SECOND } from "@core/lib";
import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount";
import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules";
interface SnapshotSchedule {
id: bigint;

6
packages/crawler/mq/exec/snapshotTick.ts
View File

@ -6,9 +6,9 @@ import {
getSnapshotsInNextSecond,
setSnapshotStatus,
videoHasProcessingSchedule
} from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { SnapshotQueue } from "mq/index.ts";
} from "db/snapshotSchedule";
import logger from "@core/log";
import { SnapshotQueue } from "mq/index";
import { sql } from "@core/db/dbNew";
const priorityMap: { [key: string]: number } = {

20
packages/crawler/mq/exec/snapshotVideo.ts
View File

@ -1,14 +1,14 @@
import { Job } from "bullmq";
import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
import logger from "@core/log/logger.ts";
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
import { insertVideoSnapshot } from "mq/task/getVideoStats.ts";
import { getSongsPublihsedAt } from "db/songs.ts";
import { getAdjustedShortTermETA } from "mq/scheduling.ts";
import { NetSchedulerError } from "@core/net/delegate.ts";
import { sql } from "@core/db/dbNew.ts";
import { closetMilestone } from "./snapshotTick.ts";
import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule";
import logger from "@core/log";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata";
import { insertVideoSnapshot } from "mq/task/getVideoStats";
import { getSongsPublihsedAt } from "db/songs";
import { getAdjustedShortTermETA } from "mq/scheduling";
import { NetSchedulerError } from "@core/net/delegate";
import { sql } from "@core/db/dbNew";
import { closetMilestone } from "./snapshotTick";
const snapshotTypeToTaskMap: { [key: string]: string } = {
milestone: "snapshotMilestoneVideo",

14
packages/crawler/mq/exec/takeBulkSnapshot.ts
View File

@ -4,14 +4,14 @@ import {
bulkSetSnapshotStatus,
scheduleSnapshot,
snapshotScheduleExists
} from "db/snapshotSchedule.ts";
import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts";
import logger from "@core/log/logger.ts";
import { NetSchedulerError } from "@core/net/delegate.ts";
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { getRegularSnapshotInterval } from "../task/regularSnapshotInterval.ts";
} from "db/snapshotSchedule";
import { bulkGetVideoStats } from "net/bulkGetVideoStats";
import logger from "@core/log";
import { NetSchedulerError } from "@core/net/delegate";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval";
import { SnapshotScheduleType } from "@core/db/schema";
import { sql } from "@core/db/dbNew.ts";
import { sql } from "@core/db/dbNew";
export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
const schedules: SnapshotScheduleType[] = job.data.schedules;

2
packages/crawler/mq/index.ts
View File

@ -1,5 +1,5 @@
import { Queue, ConnectionOptions } from "bullmq";
import { redis } from "@core/db/redis.ts";
import { redis } from "@core/db/redis";
export const LatestVideosQueue = new Queue("latestVideos", {
connection: redis as ConnectionOptions

10
packages/crawler/mq/init.ts
View File

@ -1,8 +1,8 @@
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.ts";
import logger from "@core/log/logger.ts";
import { initSnapshotWindowCounts } from "db/snapshotSchedule.ts";
import { redis } from "@core/db/redis.ts";
import { HOUR, MINUTE, SECOND } from "@core/lib";
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index";
import logger from "@core/log";
import { initSnapshotWindowCounts } from "db/snapshotSchedule";
import { redis } from "@core/db/redis";
import { sql } from "@core/db/dbNew";
export async function initMQ() {

10
packages/crawler/mq/scheduling.ts
View File

@ -1,8 +1,8 @@
import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts";
import { truncate } from "utils/truncate.ts";
import { closetMilestone } from "./exec/snapshotTick.ts";
import { HOUR, MINUTE } from "@core/const/time.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule";
import { truncate } from "utils/truncate";
import { closetMilestone } from "./exec/snapshotTick";
import { HOUR, MINUTE } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);

10
packages/crawler/mq/task/collectSongs.ts
View File

@ -1,9 +1,9 @@
import { sql } from "@core/db/dbNew";
import { aidExistsInSongs, getNotCollectedSongs } from "db/songs.ts";
import logger from "@core/log/logger.ts";
import { scheduleSnapshot } from "db/snapshotSchedule.ts";
import { MINUTE } from "@core/const/time.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { aidExistsInSongs, getNotCollectedSongs } from "db/songs";
import logger from "@core/log";
import { scheduleSnapshot } from "db/snapshotSchedule";
import { MINUTE } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
export async function collectSongs() {
const aids = await getNotCollectedSongs(sql);

14
packages/crawler/mq/task/getVideoDetails.ts
View File

@ -1,10 +1,10 @@
import { getVideoDetails } from "net/getVideoDetails.ts";
import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts";
import logger from "@core/log/logger.ts";
import { ClassifyVideoQueue } from "mq/index.ts";
import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata.ts";
import { HOUR, SECOND } from "@core/const/time.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { getVideoDetails } from "net/getVideoDetails";
import { formatTimestampToPsql } from "utils/formatTimestampToPostgre";
import logger from "@core/log";
import { ClassifyVideoQueue } from "mq/index";
import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata";
import { HOUR, SECOND } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
export async function insertVideoInfo(sql: Psql, aid: number) {
const videoExists = await videoExistsInAllData(sql, aid);

6
packages/crawler/mq/task/getVideoStats.ts
View File

@ -1,6 +1,6 @@
import { getVideoInfo } from "@core/net/getVideoInfo.ts";
import logger from "@core/log/logger.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { getVideoInfo } from "@core/net/getVideoInfo";
import logger from "@core/log";
import type { Psql } from "@core/db/psql.d";
export interface SnapshotNumber {
time: number;

14
packages/crawler/mq/task/queueLatestVideo.ts
View File

@ -1,10 +1,10 @@
import { getLatestVideoAids } from "net/getLatestVideoAids.ts";
import { videoExistsInAllData } from "db/bilibili_metadata.ts";
import { sleep } from "utils/sleep.ts";
import { SECOND } from "@core/const/time.ts";
import logger from "@core/log/logger.ts";
import { LatestVideosQueue } from "mq/index.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { getLatestVideoAids } from "net/getLatestVideoAids";
import { videoExistsInAllData } from "db/bilibili_metadata";
import { sleep } from "utils/sleep";
import { SECOND } from "@core/lib";
import logger from "@core/log";
import { LatestVideosQueue } from "mq/index";
import type { Psql } from "@core/db/psql.d";
export async function queueLatestVideos(sql: Psql): Promise<number | null> {
let page = 1;

6
packages/crawler/mq/task/regularSnapshotInterval.ts
View File

@ -1,6 +1,6 @@
import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts";
import { HOUR } from "@core/const/time.ts";
import type { Psql } from "@core/db/psql.d.ts";
import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule";
import { HOUR } from "@core/lib";
import type { Psql } from "@core/db/psql.d";
export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => {
const now = Date.now();

4
packages/crawler/mq/task/removeAllTimeoutSchedules.ts
View File

@ -1,9 +1,9 @@
import { sql } from "@core/db/dbNew";
import logger from "@core/log/logger.ts";
import logger from "@core/log";
export async function removeAllTimeoutSchedules() {
logger.log("Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker");
return await sql`
return sql`
DELETE FROM snapshot_schedule
WHERE status IN ('pending', 'processing')
AND started_at < NOW() - INTERVAL '30 minutes'

6
packages/crawler/net/bulkGetVideoStats.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate.ts";
import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d.ts";
import logger from "@core/log/logger.ts";
import networkDelegate from "@core/net/delegate";
import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
/*
* Bulk fetch video metadata from bilibili API

6
packages/crawler/net/getLatestVideoAids.ts
View File

@ -1,6 +1,6 @@
import type { VideoListResponse } from "@core/net/bilibili.d.ts";
import logger from "@core/log/logger.ts";
import networkDelegate from "@core/net/delegate.ts";
import type { VideoListResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
import networkDelegate from "@core/net/delegate";
export async function getLatestVideoAids(page: number = 1, pageSize: number = 10): Promise<number[]> {
const startFrom = 1 + pageSize * (page - 1);

6
packages/crawler/net/getVideoDetails.ts
View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate.ts";
import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d.ts";
import logger from "@core/log/logger.ts";
import networkDelegate from "@core/net/delegate";
import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d";
import logger from "@core/log";
export async function getVideoDetails(aid: number, archive: boolean = false): Promise<VideoDetailsData | null> {
const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`;

2
packages/crawler/src/bullui.ts
View File

@ -2,7 +2,7 @@ import express from "express";
import { createBullBoard } from "@bull-board/api";
import { BullMQAdapter } from "@bull-board/api/bullMQAdapter.js";
import { ExpressAdapter } from "@bull-board/express";
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.ts";
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index";
const serverAdapter = new ExpressAdapter();
serverAdapter.setBasePath("/");

12
packages/crawler/src/filterWorker.ts
View File

@ -1,10 +1,10 @@
import { ConnectionOptions, Job, Worker } from "bullmq";
import { redis } from "@core/db/redis.ts";
import logger from "@core/log/logger.ts";
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts";
import { WorkerError } from "mq/schema.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import Akari from "ml/akari.ts";
import { redis } from "@core/db/redis";
import logger from "@core/log";
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo";
import { WorkerError } from "mq/schema";
import { lockManager } from "@core/mq/lockManager";
import Akari from "ml/akari";
const shutdown = async (signal: string) => {
logger.log(`${signal} Received: Shutting down workers...`, "mq");

2
packages/crawler/src/jobAdder.ts
View File

@ -1,3 +1,3 @@
import { initMQ } from "mq/init.ts";
import { initMQ } from "mq/init";
await initMQ();

10
packages/crawler/src/worker.ts
View File

@ -11,11 +11,11 @@ import {
snapshotTickWorker,
snapshotVideoWorker,
takeBulkSnapshotForVideosWorker
} from "mq/exec/executors.ts";
import { redis } from "@core/db/redis.ts";
import logger from "@core/log/logger.ts";
import { lockManager } from "@core/mq/lockManager.ts";
import { WorkerError } from "mq/schema.ts";
} from "mq/exec/executors";
import { redis } from "@core/db/redis";
import logger from "@core/log";
import { lockManager } from "@core/mq/lockManager";
import { WorkerError } from "mq/schema";
const releaseLockForJob = async (name: string) => {
await lockManager.releaseLock(name);

120
packages/crawler/test/db/snapshotSchedule.test.ts
View File

@ -1,120 +0,0 @@
import { expect, test } from "vitest";
import { sqlTest as sql } from "@core/db/dbNew";
import { SnapshotScheduleType } from "@core/db/schema";
import { bulkSetSnapshotStatus } from "db/snapshotSchedule";
const mockSnapshotSchedules: SnapshotScheduleType[] = [
{
id: 1,
aid: 1234567890,
type: "normal",
created_at: "2025-05-04T00:00:00.000Z",
started_at: "2025-05-04T06:00:00.000Z",
finished_at: "2025-05-04T06:03:27.000Z",
status: "completed"
},
{
id: 2,
aid: 9876543210,
type: "archive",
created_at: "2025-05-03T12:00:00.000Z",
started_at: "2025-05-03T13:00:00.000Z",
finished_at: null,
status: "failed"
},
{
id: 3,
aid: 1122334455,
type: "milestone",
created_at: "2025-05-01T08:00:00.000Z",
started_at: "2025-05-01T08:12:00.000Z",
finished_at: null,
status: "processing"
}
];
const databasePreparationQuery = `
CREATE SEQUENCE "snapshot_schedule_id_seq"
START WITH 1
INCREMENT BY 1
MINVALUE 1
MAXVALUE 9223372036854775807
CACHE 1;
CREATE TABLE "snapshot_schedule"(
"id" bigint DEFAULT nextval('snapshot_schedule_id_seq'::regclass) NOT NULL,
"aid" bigint NOT NULL,
"type" text,
"created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
"started_at" timestamp with time zone,
"finished_at" timestamp with time zone,
"status" text DEFAULT 'pending'::text NOT NULL
);
CREATE INDEX idx_snapshot_schedule_aid ON snapshot_schedule USING btree (aid);
CREATE INDEX idx_snapshot_schedule_started_at ON snapshot_schedule USING btree (started_at);
CREATE INDEX idx_snapshot_schedule_status ON snapshot_schedule USING btree (status);
CREATE INDEX idx_snapshot_schedule_type ON snapshot_schedule USING btree (type);
CREATE UNIQUE INDEX snapshot_schedule_pkey ON snapshot_schedule USING btree (id);
`;
const cleanUpQuery = `
DROP SEQUENCE IF EXISTS "snapshot_schedule_id_seq" CASCADE;
DROP TABLE IF EXISTS "snapshot_schedule" CASCADE;
`;
async function testMocking() {
await sql.begin(async (tx) => {
await tx.unsafe(cleanUpQuery).simple();
await tx.unsafe(databasePreparationQuery).simple();
await tx`
INSERT INTO snapshot_schedule
${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
`;
await tx`
ROLLBACK;
`;
await tx.unsafe(cleanUpQuery).simple();
return;
});
}
async function testBulkSetSnapshotStatus() {
return await sql.begin(async (tx) => {
await tx.unsafe(cleanUpQuery).simple();
await tx.unsafe(databasePreparationQuery).simple();
await tx`
INSERT INTO snapshot_schedule
${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
`;
const ids = [1, 2, 3];
await bulkSetSnapshotStatus(tx, ids, "pending");
const rows = tx<{ status: string }[]>`
SELECT status FROM snapshot_schedule WHERE id = 1;
`.execute();
await tx`
ROLLBACK;
`;
await tx.unsafe(cleanUpQuery).simple();
return rows;
});
}
test("data mocking works", async () => {
await testMocking();
expect(() => {}).not.toThrowError();
});
test("bulkSetSnapshotStatus core logic works smoothly", async () => {
const rows = await testBulkSetSnapshotStatus();
expect(rows.every((item) => item.status === "pending")).toBe(true);
});

7
packages/crawler/tsconfig.json
View File

@ -2,17 +2,16 @@
"include": ["**/*.ts"],
"compilerOptions": {
"baseUrl": ".",
"target": "ESNext",
"module": "ESNext",
"target": "esnext",
"module": "esnext",
"useDefineForClassFields": true,
"moduleResolution": "node",
"moduleResolution": "node10",
"lib": ["ESNext", "DOM", "DOM.Iterable"],
"skipLibCheck": true,
"paths": {
"@core/*": ["../core/*"]
},
"allowSyntheticDefaultImports": true,
"allowImportingTsExtensions": true,
"noEmit": true
}
}

6
packages/crawler/vitest.config.ts
View File

@ -1,6 +0,0 @@
import { defineConfig } from "vitest/config";
import tsconfigPaths from "vite-tsconfig-paths";
export default defineConfig({
plugins: [tsconfigPaths()]
});

41
packages/elysia/routes/song/milestone.ts Normal file
View File

@ -0,0 +1,41 @@
import { Elysia, t } from "elysia";
import { dbMain } from "@core/drizzle";
import { bilibiliMetadata, latestVideoSnapshot } from "@core/drizzle/main/schema";
import { eq, and, gte, lt, desc } from "drizzle-orm";
type MileStoneType = "dendou" | "densetsu" | "shinwa";
const range = {
dendou: [90000, 99999],
densetsu: [900000, 999999],
shinwa: [5000000, 9999999]
};
export const closeMileStoneHandler = new Elysia({ prefix: "/song" }).get(
"/close-milestone/:type",
async (c) => {
const type = c.params.type;
const min = range[type as MileStoneType][0];
const max = range[type as MileStoneType][1];
const data = await dbMain
.select()
.from(bilibiliMetadata)
.innerJoin(latestVideoSnapshot, eq(latestVideoSnapshot.aid, bilibiliMetadata.aid))
.where(and(gte(latestVideoSnapshot.views, min), lt(latestVideoSnapshot.views, max)))
.orderBy(desc(latestVideoSnapshot.views));
const aids = data.map((song) => song.bilibili_metadata.aid);
for (const aid of aids) {
}
return data;
},
{
response: {
200: t.Array(t.Any()),
404: t.Object({
message: t.String()
})
}
}
);

2
packages/elysia/src/index.ts
View File

@ -15,9 +15,7 @@ const app = new Elysia({
hostname: host
}
})
// @ts-expect-error idfk
.use(cors())
// @ts-expect-error idfk
.use(openapi())
.use(rootHandler)
.use(pingHandler)

5
packages/next/lib/net.ts
View File

@ -1,4 +1,4 @@
import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios";
import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, Method } from "axios";
export class ApiRequestError extends Error {
public code: number | undefined;
@ -49,8 +49,7 @@ export async function fetcher<JSON = unknown>(
const response = await httpMethods[m](url, data, fullConfig);
return response.data;
} else if (m === "delete") {
const response = await axios.delete(url, fullConfig);
return response;
return await axios.delete(url, fullConfig);
} else {
const response = await httpMethods[m](url, fullConfig);
return response.data;

5
packages/solid/src/lib/net.ts
View File

@ -1,4 +1,4 @@
import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios";
import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, Method } from "axios";
export class ApiRequestError extends Error {
public code: number | undefined;
@ -49,8 +49,7 @@ export async function fetcher<JSON = unknown>(
const response = await httpMethods[m](url, data, fullConfig);
return response.data;
} else if (m === "delete") {
const response = await axios.delete(url, fullConfig);
return response;
return await axios.delete(url, fullConfig);
} else {
const response = await httpMethods[m](url, fullConfig);
return response.data;

1
packages/temp_frontend/app/routes/home.tsx
View File

@ -1,7 +1,6 @@
import type { Route } from "./+types/home";
import { treaty } from "@elysiajs/eden";
import type { App } from "@elysia/src";
import { toast } from "sonner";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { SearchIcon } from "@/components/icons/search";

2
src/fillSongInfo.ts
View File

@ -1,5 +1,5 @@
import arg from "arg";
import logger from "@core/log/logger";
import logger from "@core/log";
import { sql } from "@core/index";
import type { Row } from "postgres";

2
src/fixCover.ts
View File

@ -1,6 +1,6 @@
import arg from "arg";
import { Database } from "bun:sqlite";
import logger from "@core/log/logger";
import logger from "@core/log";
import type { VideoDetailsData } from "@core/net/bilibili.d.ts";
import { sql } from "@core/index";

2
src/fixPubDate.ts
View File

@ -1,6 +1,6 @@
import arg from "arg";
import { Database } from "bun:sqlite";
import logger from "@core/log/logger";
import logger from "@core/log";
import type { VideoDetailsData } from "@core/net/bilibili.d.ts";
import { sql } from "@core/index";