Compare commits
22 Commits
Author | SHA1 | Date | |
---|---|---|---|
92c3c8eefe | |||
497ea031d8 | |||
39ca394a56 | |||
0bd1771f35 | |||
328c73c209 | |||
5ac952ec13 | |||
2cf5923b28 | |||
75973c72ee | |||
b40d24721c | |||
0a6ecc6314 | |||
b4a0320e3e | |||
8cf9395354 | |||
1e8d28e194 | |||
c0340677a1 | |||
54a2de0a11 | |||
2c83b79881 | |||
1a20d5afe0 | |||
2b0497c83a | |||
3bc72720d1 | |||
1ff71ab241 | |||
cf7a285f57 | |||
79a37d927a |
17
package.json
17
package.json
@ -1,12 +1,23 @@
|
||||
{
|
||||
"name": "cvsa",
|
||||
"version": "2.13.22",
|
||||
"private": true,
|
||||
"version": "3.15.34",
|
||||
"private": false,
|
||||
"type": "module",
|
||||
"workspaces": [
|
||||
"packages/frontend",
|
||||
"packages/core",
|
||||
"packages/backend",
|
||||
"packages/crawler"
|
||||
]
|
||||
],
|
||||
"dependencies": {
|
||||
"arg": "^5.0.2",
|
||||
"postgres": "^3.4.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.2.15",
|
||||
"prettier": "^3.5.3",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"vitest": "^3.1.2",
|
||||
"vitest-tsconfig-paths": "^3.4.1"
|
||||
}
|
||||
}
|
||||
|
13
packages/backend/db/latestSnapshots.ts
Normal file
13
packages/backend/db/latestSnapshots.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import { sql } from "@core/db/dbNew";
|
||||
import type { LatestSnapshotType } from "@core/db/schema.d.ts";
|
||||
|
||||
export async function getVideosInViewsRange(minViews: number, maxViews: number) {
|
||||
return sql<LatestSnapshotType[]>`
|
||||
SELECT *
|
||||
FROM latest_video_snapshot
|
||||
WHERE views >= ${minViews}
|
||||
AND views <= ${maxViews}
|
||||
ORDER BY views DESC
|
||||
LIMIT 5000
|
||||
`;
|
||||
}
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@cvsa/backend",
|
||||
"private": false,
|
||||
"version": "0.5.3",
|
||||
"version": "0.6.0",
|
||||
"scripts": {
|
||||
"format": "prettier --write .",
|
||||
"dev": "NODE_ENV=development bun run --hot src/main.ts",
|
||||
|
75
packages/backend/routes/session/[id]/DELETE.ts
Normal file
75
packages/backend/routes/session/[id]/DELETE.ts
Normal file
@ -0,0 +1,75 @@
|
||||
import { Context } from "hono";
|
||||
import { Bindings, BlankEnv } from "hono/types";
|
||||
import { ErrorResponse } from "src/schema";
|
||||
import { createHandlers } from "src/utils.ts";
|
||||
import { sqlCred } from "@core/db/dbNew";
|
||||
import { object, string, ValidationError } from "yup";
|
||||
import { setCookie } from "hono/cookie";
|
||||
|
||||
const loginSessionExists = async (sessionID: string) => {
|
||||
const result = await sqlCred`
|
||||
SELECT 1
|
||||
FROM login_sessions
|
||||
WHERE id = ${sessionID}
|
||||
`;
|
||||
return result.length > 0;
|
||||
};
|
||||
|
||||
export const logoutHandler = createHandlers(async (c: Context<BlankEnv & { Bindings: Bindings }, "/session/:id">) => {
|
||||
try {
|
||||
const session_id = c.req.param("id");
|
||||
|
||||
const exists = loginSessionExists(session_id);
|
||||
|
||||
if (!exists) {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "Cannot found given session_id.",
|
||||
errors: [`Session ${session_id} not found`],
|
||||
code: "ENTITY_NOT_FOUND"
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 404);
|
||||
}
|
||||
|
||||
await sqlCred`
|
||||
UPDATE login_sessions
|
||||
SET deactivated_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ${session_id}
|
||||
`;
|
||||
|
||||
const isDev = process.env.NODE_ENV === "development";
|
||||
|
||||
setCookie(c, "session_id", "", {
|
||||
path: "/",
|
||||
maxAge: 0,
|
||||
domain: process.env.DOMAIN,
|
||||
secure: isDev ? true : true,
|
||||
sameSite: isDev ? "None" : "Lax",
|
||||
httpOnly: true
|
||||
});
|
||||
|
||||
return c.body(null, 204);
|
||||
} catch (e) {
|
||||
if (e instanceof ValidationError) {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "Invalid registration data.",
|
||||
errors: e.errors,
|
||||
code: "INVALID_PAYLOAD"
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 400);
|
||||
} else if (e instanceof SyntaxError) {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "Invalid JSON payload.",
|
||||
errors: [e.message],
|
||||
code: "INVALID_FORMAT"
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 400);
|
||||
} else {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "Unknown error.",
|
||||
errors: [(e as Error).message],
|
||||
code: "UNKNOWN_ERROR"
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 500);
|
||||
}
|
||||
}
|
||||
});
|
1
packages/backend/routes/session/index.ts
Normal file
1
packages/backend/routes/session/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from "./[id]/DELETE";
|
65
packages/backend/routes/videos/GET.ts
Normal file
65
packages/backend/routes/videos/GET.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import type { Context } from "hono";
|
||||
import { createHandlers } from "src/utils.ts";
|
||||
import type { BlankEnv, BlankInput } from "hono/types";
|
||||
import { number, object, ValidationError } from "yup";
|
||||
import { ErrorResponse } from "src/schema";
|
||||
import { startTime, endTime } from "hono/timing";
|
||||
import { getVideosInViewsRange } from "@/db/latestSnapshots";
|
||||
|
||||
const SnapshotQueryParamsSchema = object({
|
||||
min_views: number().integer().optional().positive(),
|
||||
max_views: number().integer().optional().positive()
|
||||
});
|
||||
|
||||
type ContextType = Context<BlankEnv, "/videos", BlankInput>;
|
||||
|
||||
export const getVideosHanlder = createHandlers(async (c: ContextType) => {
|
||||
startTime(c, "parse", "Parse the request");
|
||||
try {
|
||||
const queryParams = await SnapshotQueryParamsSchema.validate(c.req.query());
|
||||
const { min_views, max_views } = queryParams;
|
||||
|
||||
if (!min_views && !max_views) {
|
||||
const response: ErrorResponse<string> = {
|
||||
code: "INVALID_QUERY_PARAMS",
|
||||
message: "Invalid query parameters",
|
||||
errors: ["Must provide one of these query parameters: min_views, max_views"]
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 400);
|
||||
}
|
||||
|
||||
endTime(c, "parse");
|
||||
|
||||
startTime(c, "db", "Query the database");
|
||||
|
||||
const minViews = min_views ? min_views : 0;
|
||||
const maxViews = max_views ? max_views : 2147483647;
|
||||
|
||||
const result = await getVideosInViewsRange(minViews, maxViews);
|
||||
|
||||
endTime(c, "db");
|
||||
|
||||
const rows = result.map((row) => ({
|
||||
...row,
|
||||
aid: Number(row.aid)
|
||||
}));
|
||||
|
||||
return c.json(rows);
|
||||
} catch (e: unknown) {
|
||||
if (e instanceof ValidationError) {
|
||||
const response: ErrorResponse<string> = {
|
||||
code: "INVALID_QUERY_PARAMS",
|
||||
message: "Invalid query parameters",
|
||||
errors: e.errors
|
||||
};
|
||||
return c.json<ErrorResponse<string>>(response, 400);
|
||||
} else {
|
||||
const response: ErrorResponse<unknown> = {
|
||||
code: "UNKNOWN_ERROR",
|
||||
message: "Unhandled error",
|
||||
errors: [e]
|
||||
};
|
||||
return c.json<ErrorResponse<unknown>>(response, 500);
|
||||
}
|
||||
}
|
||||
});
|
1
packages/backend/routes/videos/index.ts
Normal file
1
packages/backend/routes/videos/index.ts
Normal file
@ -0,0 +1 @@
|
||||
export * from "./GET.ts";
|
@ -15,4 +15,4 @@ configureRoutes(app);
|
||||
|
||||
await startServer(app);
|
||||
|
||||
export const VERSION = "0.5.2";
|
||||
export const VERSION = "0.6.0";
|
||||
|
@ -6,17 +6,23 @@ import { Hono } from "hono";
|
||||
import { Variables } from "hono/types";
|
||||
import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha";
|
||||
import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET.ts";
|
||||
import { getVideosHanlder } from "@/routes/videos";
|
||||
import { loginHandler } from "@/routes/login/session/POST";
|
||||
import { logoutHandler } from "@/routes/session";
|
||||
|
||||
export function configureRoutes(app: Hono<{ Variables: Variables }>) {
|
||||
app.get("/", ...rootHandler);
|
||||
app.all("/ping", ...pingHandler);
|
||||
|
||||
app.get("/videos", ...getVideosHanlder);
|
||||
|
||||
app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
|
||||
app.get("/video/:id/info", ...videoInfoHandler);
|
||||
|
||||
app.post("/login/session", ...loginHandler);
|
||||
|
||||
app.delete("/session/:id", ...logoutHandler);
|
||||
|
||||
app.post("/user", ...registerHandler);
|
||||
app.get("/user/session/:id", ...getUserByLoginSessionHandler);
|
||||
|
||||
|
9
packages/core/db/schema.d.ts
vendored
9
packages/core/db/schema.d.ts
vendored
@ -8,7 +8,7 @@ export interface BiliUserType {
|
||||
|
||||
export interface VideoSnapshotType {
|
||||
id: number;
|
||||
created_at: string;
|
||||
created_at: Date;
|
||||
views: number;
|
||||
coins: number;
|
||||
likes: number;
|
||||
@ -35,9 +35,9 @@ export interface SnapshotScheduleType {
|
||||
id: number;
|
||||
aid: number;
|
||||
type?: string;
|
||||
created_at: string;
|
||||
started_at?: string;
|
||||
finished_at?: string;
|
||||
created_at: Date;
|
||||
started_at?: Date;
|
||||
finished_at?: Date;
|
||||
status: string;
|
||||
}
|
||||
|
||||
@ -48,6 +48,7 @@ export interface UserType {
|
||||
password: string;
|
||||
unq_id: string;
|
||||
role: string;
|
||||
created_at: Date;
|
||||
}
|
||||
|
||||
export interface BiliVideoMetadataType {
|
||||
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "@cvsa/core",
|
||||
"private": false,
|
||||
"version": "0.0.7",
|
||||
"version": "0.0.10",
|
||||
"scripts": {
|
||||
"test": "bun --env-file=.env.test run vitest",
|
||||
"build": "bun build ./index.ts --target node --outdir ./dist"
|
||||
|
@ -30,9 +30,9 @@ export async function insertVideoLabel(sql: Psql, aid: number, label: number) {
|
||||
}
|
||||
|
||||
export async function getVideoInfoFromAllData(sql: Psql, aid: number) {
|
||||
const rows = await sql<BiliVideoMetadataType[]>`
|
||||
const rows = await sql<AllDataType[]>`
|
||||
SELECT * FROM bilibili_metadata WHERE aid = ${aid}
|
||||
`;
|
||||
`;
|
||||
const row = rows[0];
|
||||
let authorInfo = "";
|
||||
if (row.uid && (await userExistsInBiliUsers(sql, row.uid))) {
|
||||
|
@ -4,25 +4,25 @@ import type { Psql } from "@core/db/psql.d.ts";
|
||||
|
||||
export async function getVideosNearMilestone(sql: Psql) {
|
||||
const queryResult = await sql<LatestSnapshotType[]>`
|
||||
SELECT ls.*
|
||||
SELECT ls.*
|
||||
FROM latest_video_snapshot ls
|
||||
RIGHT JOIN songs ON songs.aid = ls.aid
|
||||
RIGHT JOIN songs ON songs.aid = ls.aid
|
||||
WHERE
|
||||
(views >= 50000 AND views < 100000) OR
|
||||
(views >= 900000 AND views < 1000000) OR
|
||||
(views >= 9900000 AND views < 10000000)
|
||||
(views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
|
||||
UNION
|
||||
SELECT ls.*
|
||||
FROM latest_video_snapshot ls
|
||||
WHERE
|
||||
(views >= 90000 AND views < 100000) OR
|
||||
(views >= 900000 AND views < 1000000) OR
|
||||
(views >= 9900000 AND views < 10000000)
|
||||
(views >= CEIL(views::float/1000000::float)*1000000-100000 AND views < CEIL(views::float/1000000::float)*1000000)
|
||||
`;
|
||||
return queryResult.map((row) => {
|
||||
return {
|
||||
...row,
|
||||
aid: Number(row.aid),
|
||||
aid: Number(row.aid)
|
||||
};
|
||||
});
|
||||
}
|
||||
@ -40,7 +40,7 @@ export async function getLatestVideoSnapshot(sql: Psql, aid: number): Promise<nu
|
||||
return {
|
||||
...row,
|
||||
aid: Number(row.aid),
|
||||
time: new Date(row.time).getTime(),
|
||||
time: new Date(row.time).getTime()
|
||||
};
|
||||
})[0];
|
||||
}
|
||||
|
@ -19,8 +19,8 @@ export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: Redis)
|
||||
const startTime = now.getTime();
|
||||
|
||||
const result = await sql<{ window_start: Date; count: number }[]>`
|
||||
SELECT
|
||||
date_trunc('hour', started_at) +
|
||||
SELECT
|
||||
date_trunc('hour', started_at) +
|
||||
(EXTRACT(minute FROM started_at)::int / 5 * INTERVAL '5 minutes') AS window_start,
|
||||
COUNT(*) AS count
|
||||
FROM snapshot_schedule
|
||||
@ -56,30 +56,18 @@ async function getWindowCount(redisClient: Redis, offset: number): Promise<numbe
|
||||
|
||||
export async function snapshotScheduleExists(sql: Psql, id: number) {
|
||||
const rows = await sql<{ id: number }[]>`
|
||||
SELECT id
|
||||
FROM snapshot_schedule
|
||||
SELECT id
|
||||
FROM snapshot_schedule
|
||||
WHERE id = ${id}
|
||||
`;
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
export async function videoHasActiveSchedule(sql: Psql, aid: number) {
|
||||
const rows = await sql<{ status: string }[]>`
|
||||
SELECT status
|
||||
FROM snapshot_schedule
|
||||
WHERE aid = ${aid}
|
||||
AND (status = 'pending'
|
||||
OR status = 'processing'
|
||||
)
|
||||
`
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
export async function videoHasActiveScheduleWithType(sql: Psql, aid: number, type: string) {
|
||||
const rows = await sql<{ status: string }[]>`
|
||||
SELECT status FROM snapshot_schedule
|
||||
SELECT status FROM snapshot_schedule
|
||||
WHERE aid = ${aid}
|
||||
AND (status = 'pending' OR status = 'processing')
|
||||
AND (status = 'pending' OR status = 'processing')
|
||||
AND type = ${type}
|
||||
`;
|
||||
return rows.length > 0;
|
||||
@ -88,10 +76,10 @@ export async function videoHasActiveScheduleWithType(sql: Psql, aid: number, typ
|
||||
export async function videoHasProcessingSchedule(sql: Psql, aid: number) {
|
||||
const rows = await sql<{ status: string }[]>`
|
||||
SELECT status
|
||||
FROM snapshot_schedule
|
||||
FROM snapshot_schedule
|
||||
WHERE aid = ${aid}
|
||||
AND status = 'processing'
|
||||
`
|
||||
`;
|
||||
return rows.length > 0;
|
||||
}
|
||||
|
||||
@ -100,9 +88,9 @@ export async function bulkGetVideosWithoutProcessingSchedules(sql: Psql, aids: n
|
||||
SELECT aid
|
||||
FROM snapshot_schedule
|
||||
WHERE aid = ANY(${aids})
|
||||
AND status != 'processing'
|
||||
AND status != 'processing'
|
||||
GROUP BY aid
|
||||
`
|
||||
`;
|
||||
return rows.map((row) => Number(row.aid));
|
||||
}
|
||||
|
||||
@ -146,8 +134,8 @@ export async function findSnapshotBefore(sql: Psql, aid: number, targetTime: Dat
|
||||
|
||||
export async function hasAtLeast2Snapshots(sql: Psql, aid: number) {
|
||||
const res = await sql<{ count: number }[]>`
|
||||
SELECT COUNT(*)
|
||||
FROM video_snapshot
|
||||
SELECT COUNT(*)
|
||||
FROM video_snapshot
|
||||
WHERE aid = ${aid}
|
||||
`;
|
||||
return res[0].count >= 2;
|
||||
@ -155,10 +143,10 @@ export async function hasAtLeast2Snapshots(sql: Psql, aid: number) {
|
||||
|
||||
export async function getLatestSnapshot(sql: Psql, aid: number): Promise<Snapshot | null> {
|
||||
const res = await sql<{ created_at: string; views: number }[]>`
|
||||
SELECT created_at, views
|
||||
FROM video_snapshot
|
||||
SELECT created_at, views
|
||||
FROM video_snapshot
|
||||
WHERE aid = ${aid}
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
if (res.length === 0) return null;
|
||||
@ -194,7 +182,8 @@ export async function scheduleSnapshot(
|
||||
aid: number,
|
||||
type: string,
|
||||
targetTime: number,
|
||||
force: boolean = false
|
||||
force: boolean = false,
|
||||
adjustTime: boolean = true
|
||||
) {
|
||||
let adjustedTime = new Date(targetTime);
|
||||
const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type);
|
||||
@ -216,16 +205,16 @@ export async function scheduleSnapshot(
|
||||
}
|
||||
}
|
||||
if (hashActiveSchedule && !force) return;
|
||||
if (type !== "milestone" && type !== "new") {
|
||||
if (type !== "milestone" && type !== "new" && adjustTime) {
|
||||
adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis);
|
||||
}
|
||||
logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot");
|
||||
return sql`
|
||||
INSERT INTO snapshot_schedule
|
||||
(aid, type, started_at)
|
||||
INSERT INTO snapshot_schedule
|
||||
(aid, type, started_at)
|
||||
VALUES (
|
||||
${aid},
|
||||
${type},
|
||||
${aid},
|
||||
${type},
|
||||
${adjustedTime.toISOString()}
|
||||
)
|
||||
`;
|
||||
@ -236,10 +225,11 @@ export async function bulkScheduleSnapshot(
|
||||
aids: number[],
|
||||
type: string,
|
||||
targetTime: number,
|
||||
force: boolean = false
|
||||
force: boolean = false,
|
||||
adjustTime: boolean = true
|
||||
) {
|
||||
for (const aid of aids) {
|
||||
await scheduleSnapshot(sql, aid, type, targetTime, force);
|
||||
await scheduleSnapshot(sql, aid, type, targetTime, force, adjustTime);
|
||||
}
|
||||
}
|
||||
|
||||
@ -292,23 +282,23 @@ export async function adjustSnapshotTime(
|
||||
}
|
||||
|
||||
export async function getSnapshotsInNextSecond(sql: Psql) {
|
||||
const rows = await sql<SnapshotScheduleType[]>`
|
||||
SELECT *
|
||||
FROM snapshot_schedule
|
||||
WHERE started_at <= NOW() + INTERVAL '1 seconds' AND status = 'pending' AND type != 'normal'
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN type = 'milestone' THEN 0
|
||||
ELSE 1
|
||||
END,
|
||||
started_at
|
||||
LIMIT 10;
|
||||
`
|
||||
return rows;
|
||||
return sql<SnapshotScheduleType[]>`
|
||||
SELECT *
|
||||
FROM snapshot_schedule
|
||||
WHERE started_at <= NOW() + INTERVAL '1 seconds'
|
||||
AND status = 'pending'
|
||||
AND type != 'normal'
|
||||
ORDER BY CASE
|
||||
WHEN type = 'milestone' THEN 0
|
||||
ELSE 1
|
||||
END,
|
||||
started_at
|
||||
LIMIT 10;
|
||||
`;
|
||||
}
|
||||
|
||||
export async function getBulkSnapshotsInNextSecond(sql: Psql) {
|
||||
const rows = await sql<SnapshotScheduleType[]>`
|
||||
return sql<SnapshotScheduleType[]>`
|
||||
SELECT *
|
||||
FROM snapshot_schedule
|
||||
WHERE (started_at <= NOW() + INTERVAL '15 seconds')
|
||||
@ -320,38 +310,34 @@ export async function getBulkSnapshotsInNextSecond(sql: Psql) {
|
||||
END,
|
||||
started_at
|
||||
LIMIT 1000;
|
||||
`
|
||||
return rows;
|
||||
`;
|
||||
}
|
||||
|
||||
export async function setSnapshotStatus(sql: Psql, id: number, status: string) {
|
||||
return await sql`
|
||||
UPDATE snapshot_schedule SET status = ${status} WHERE id = ${id}
|
||||
return sql`
|
||||
UPDATE snapshot_schedule
|
||||
SET status = ${status}
|
||||
WHERE id = ${id}
|
||||
`;
|
||||
}
|
||||
|
||||
export async function bulkSetSnapshotStatus(sql: Psql, ids: number[], status: string) {
|
||||
return await sql`
|
||||
UPDATE snapshot_schedule SET status = ${status} WHERE id = ANY(${ids})
|
||||
return sql`
|
||||
UPDATE snapshot_schedule
|
||||
SET status = ${status}
|
||||
WHERE id = ANY (${ids})
|
||||
`;
|
||||
}
|
||||
|
||||
export async function getVideosWithoutActiveSnapshotSchedule(sql: Psql) {
|
||||
export async function getVideosWithoutActiveSnapshotScheduleByType(sql: Psql, type: string) {
|
||||
const rows = await sql<{ aid: string }[]>`
|
||||
SELECT s.aid
|
||||
FROM songs s
|
||||
LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing')
|
||||
LEFT JOIN snapshot_schedule ss ON
|
||||
s.aid = ss.aid AND
|
||||
(ss.status = 'pending' OR ss.status = 'processing') AND
|
||||
ss.type = ${type}
|
||||
WHERE ss.aid IS NULL
|
||||
`;
|
||||
return rows.map((r) => Number(r.aid));
|
||||
}
|
||||
|
||||
export async function getAllVideosWithoutActiveSnapshotSchedule(psql: Psql) {
|
||||
const rows = await psql<{ aid: number }[]>`
|
||||
SELECT s.aid
|
||||
FROM bilibili_metadata s
|
||||
LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing')
|
||||
WHERE ss.aid IS NULL
|
||||
`
|
||||
return rows.map((r) => Number(r.aid));
|
||||
}
|
||||
|
@ -2,7 +2,7 @@ import type { Psql } from "@core/db/psql.d.ts";
|
||||
import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts";
|
||||
|
||||
export async function getNotCollectedSongs(sql: Psql) {
|
||||
const rows = await sql<{ aid: number }[]>`
|
||||
const rows = await sql<{ aid: number }[]>`
|
||||
SELECT lr.aid
|
||||
FROM labelling_result lr
|
||||
WHERE lr.label != 0
|
||||
@ -12,28 +12,28 @@ export async function getNotCollectedSongs(sql: Psql) {
|
||||
WHERE s.aid = lr.aid
|
||||
);
|
||||
`;
|
||||
return rows.map((row) => row.aid);
|
||||
return rows.map((row) => row.aid);
|
||||
}
|
||||
|
||||
export async function aidExistsInSongs(sql: Psql, aid: number) {
|
||||
const rows = await sql<{ exists: boolean }[]>`
|
||||
const rows = await sql<{ exists: boolean }[]>`
|
||||
SELECT EXISTS (
|
||||
SELECT 1
|
||||
FROM songs
|
||||
WHERE aid = ${aid}
|
||||
);
|
||||
`;
|
||||
return rows[0].exists;
|
||||
return rows[0].exists;
|
||||
}
|
||||
|
||||
export async function getSongsPublihsedAt(sql: Psql, aid: number) {
|
||||
const rows = await sql<{ published_at: string }[]>`
|
||||
const rows = await sql<{ published_at: string }[]>`
|
||||
SELECT published_at
|
||||
FROM songs
|
||||
WHERE aid = ${aid};
|
||||
`;
|
||||
if (rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return parseTimestampFromPsql(rows[0].published_at);
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
return null;
|
||||
}
|
||||
return parseTimestampFromPsql(rows[0].published_at);
|
||||
}
|
||||
|
@ -16,8 +16,8 @@ class AkariProto extends AIManager {
|
||||
constructor() {
|
||||
super();
|
||||
this.models = {
|
||||
"classifier": onnxClassifierPath,
|
||||
"embedding": onnxEmbeddingPath,
|
||||
classifier: onnxClassifierPath,
|
||||
embedding: onnxEmbeddingPath
|
||||
};
|
||||
}
|
||||
|
||||
@ -55,7 +55,7 @@ class AkariProto extends AIManager {
|
||||
|
||||
const { input_ids } = await tokenizer(texts, {
|
||||
add_special_tokens: false,
|
||||
return_tensor: false,
|
||||
return_tensor: false
|
||||
});
|
||||
|
||||
const cumsum = (arr: number[]): number[] =>
|
||||
@ -66,9 +66,9 @@ class AkariProto extends AIManager {
|
||||
|
||||
const inputs = {
|
||||
input_ids: new ort.Tensor("int64", new BigInt64Array(flattened_input_ids.map(BigInt)), [
|
||||
flattened_input_ids.length,
|
||||
flattened_input_ids.length
|
||||
]),
|
||||
offsets: new ort.Tensor("int64", new BigInt64Array(offsets.map(BigInt)), [offsets.length]),
|
||||
offsets: new ort.Tensor("int64", new BigInt64Array(offsets.map(BigInt)), [offsets.length])
|
||||
};
|
||||
|
||||
const { embeddings } = await session.run(inputs);
|
||||
@ -77,21 +77,14 @@ class AkariProto extends AIManager {
|
||||
|
||||
private async runClassification(embeddings: number[]): Promise<number[]> {
|
||||
const session = this.getModelSession("classifier");
|
||||
const inputTensor = new ort.Tensor(
|
||||
Float32Array.from(embeddings),
|
||||
[1, 3, 1024],
|
||||
);
|
||||
const inputTensor = new ort.Tensor(Float32Array.from(embeddings), [1, 3, 1024]);
|
||||
|
||||
const { logits } = await session.run({ channel_features: inputTensor });
|
||||
return this.softmax(logits.data as Float32Array);
|
||||
}
|
||||
|
||||
public async classifyVideo(title: string, description: string, tags: string, aid?: number): Promise<number> {
|
||||
const embeddings = await this.getJinaEmbeddings1024([
|
||||
title,
|
||||
description,
|
||||
tags,
|
||||
]);
|
||||
const embeddings = await this.getJinaEmbeddings1024([title, description, tags]);
|
||||
const probabilities = await this.runClassification(embeddings);
|
||||
if (aid) {
|
||||
logger.log(`Prediction result for aid: ${aid}: [${probabilities.map((p) => p.toFixed(5))}]`, "ml");
|
||||
|
@ -1 +1 @@
|
||||
export const AkariModelVersion = "3.17";
|
||||
export const AkariModelVersion = "3.17";
|
||||
|
@ -6,8 +6,7 @@ export class AIManager {
|
||||
public sessions: { [key: string]: ort.InferenceSession } = {};
|
||||
public models: { [key: string]: string } = {};
|
||||
|
||||
constructor() {
|
||||
}
|
||||
constructor() {}
|
||||
|
||||
public async init() {
|
||||
const modelKeys = Object.keys(this.models);
|
||||
|
@ -1,11 +1,28 @@
|
||||
import { Job } from "bullmq";
|
||||
import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
|
||||
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
import { lockManager } from "@core/mq/lockManager.ts";
|
||||
import { getLatestVideoSnapshot } from "db/snapshot.ts";
|
||||
import { HOUR, MINUTE } from "@core/const/time.ts";
|
||||
import { MINUTE } from "@core/const/time.ts";
|
||||
import { sql } from "@core/db/dbNew";
|
||||
|
||||
function getNextSaturdayMidnightTimestamp(): number {
|
||||
const now = new Date();
|
||||
const currentDay = now.getDay();
|
||||
|
||||
let daysUntilNextSaturday = (6 - currentDay + 7) % 7;
|
||||
|
||||
if (daysUntilNextSaturday === 0) {
|
||||
daysUntilNextSaturday = 7;
|
||||
}
|
||||
|
||||
const nextSaturday = new Date(now);
|
||||
nextSaturday.setDate(nextSaturday.getDate() + daysUntilNextSaturday);
|
||||
nextSaturday.setHours(0, 0, 0, 0);
|
||||
|
||||
return nextSaturday.getTime();
|
||||
}
|
||||
|
||||
export const archiveSnapshotsWorker = async (_job: Job) => {
|
||||
try {
|
||||
const startedAt = Date.now();
|
||||
@ -14,21 +31,22 @@ export const archiveSnapshotsWorker = async (_job: Job) => {
|
||||
return;
|
||||
}
|
||||
await lockManager.acquireLock("dispatchArchiveSnapshots", 30 * 60);
|
||||
const aids = await getAllVideosWithoutActiveSnapshotSchedule(sql);
|
||||
const aids = await getVideosWithoutActiveSnapshotScheduleByType(sql, "archive");
|
||||
for (const rawAid of aids) {
|
||||
const aid = Number(rawAid);
|
||||
const latestSnapshot = await getLatestVideoSnapshot(sql, aid);
|
||||
const now = Date.now();
|
||||
const lastSnapshotedAt = latestSnapshot?.time ?? now;
|
||||
const interval = 168;
|
||||
const nextSatMidnight = getNextSaturdayMidnightTimestamp();
|
||||
const interval = nextSatMidnight - now;
|
||||
logger.log(
|
||||
`Scheduled archive snapshot for aid ${aid} in ${interval} hours.`,
|
||||
"mq",
|
||||
"fn:archiveSnapshotsWorker"
|
||||
);
|
||||
const targetTime = lastSnapshotedAt + interval * HOUR;
|
||||
const targetTime = lastSnapshotedAt + interval;
|
||||
await scheduleSnapshot(sql, aid, "archive", targetTime);
|
||||
if (now - startedAt > 250 * MINUTE) {
|
||||
if (now - startedAt > 30 * MINUTE) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
@ -34,7 +34,7 @@ export const classifyVideoWorker = async (job: Job) => {
|
||||
|
||||
await job.updateData({
|
||||
...job.data,
|
||||
label: label,
|
||||
label: label
|
||||
});
|
||||
|
||||
return 0;
|
||||
@ -46,19 +46,19 @@ export const classifyVideosWorker = async () => {
|
||||
return;
|
||||
}
|
||||
|
||||
await lockManager.acquireLock("classifyVideos");
|
||||
await lockManager.acquireLock("classifyVideos", 5 * 60);
|
||||
|
||||
const videos = await getUnlabelledVideos(sql);
|
||||
logger.log(`Found ${videos.length} unlabelled videos`);
|
||||
|
||||
let i = 0;
|
||||
const startTime = new Date().getTime();
|
||||
for (const aid of videos) {
|
||||
if (i > 200) {
|
||||
const now = new Date().getTime();
|
||||
if (now - startTime > 4.2 * MINUTE) {
|
||||
await lockManager.releaseLock("classifyVideos");
|
||||
return 10000 + i;
|
||||
return 1;
|
||||
}
|
||||
await ClassifyVideoQueue.add("classifyVideo", { aid: Number(aid) });
|
||||
i++;
|
||||
}
|
||||
await lockManager.releaseLock("classifyVideos");
|
||||
return 0;
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { Job } from "bullmq";
|
||||
import { collectSongs } from "mq/task/collectSongs.ts";
|
||||
|
||||
export const collectSongsWorker = async (_job: Job): Promise<void> =>{
|
||||
export const collectSongsWorker = async (_job: Job): Promise<void> => {
|
||||
await collectSongs();
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
@ -16,8 +16,8 @@ export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {
|
||||
if (eta > 144) continue;
|
||||
const now = Date.now();
|
||||
const scheduledNextSnapshotDelay = eta * HOUR;
|
||||
const maxInterval = 1 * HOUR;
|
||||
const minInterval = 1 * SECOND;
|
||||
const maxInterval = 1.2 * HOUR;
|
||||
const minInterval = 2 * SECOND;
|
||||
const delay = truncate(scheduledNextSnapshotDelay, minInterval, maxInterval);
|
||||
const targetTime = now + delay;
|
||||
await scheduleSnapshot(sql, aid, "milestone", targetTime);
|
||||
@ -25,5 +25,5 @@ export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => {
|
||||
}
|
||||
} catch (e) {
|
||||
logger.error(e as Error, "mq", "fn:dispatchMilestoneSnapshotsWorker");
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { Job } from "bullmq";
|
||||
import { getLatestVideoSnapshot } from "db/snapshot.ts";
|
||||
import { truncate } from "utils/truncate.ts";
|
||||
import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts";
|
||||
import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
import { HOUR, MINUTE, WEEK } from "@core/const/time.ts";
|
||||
import { lockManager } from "@core/mq/lockManager.ts";
|
||||
@ -17,7 +17,7 @@ export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise<void> =
|
||||
}
|
||||
await lockManager.acquireLock("dispatchRegularSnapshots", 30 * 60);
|
||||
|
||||
const aids = await getVideosWithoutActiveSnapshotSchedule(sql);
|
||||
const aids = await getVideosWithoutActiveSnapshotScheduleByType(sql, "normal");
|
||||
for (const rawAid of aids) {
|
||||
const aid = Number(rawAid);
|
||||
const latestSnapshot = await getLatestVideoSnapshot(sql, aid);
|
||||
|
@ -7,4 +7,4 @@ export * from "./dispatchMilestoneSnapshots.ts";
|
||||
export * from "./dispatchRegularSnapshots.ts";
|
||||
export * from "./snapshotVideo.ts";
|
||||
export * from "./scheduleCleanup.ts";
|
||||
export * from "./snapshotTick.ts";
|
||||
export * from "./snapshotTick.ts";
|
||||
|
@ -2,6 +2,6 @@ import { sql } from "@core/db/dbNew";
|
||||
import { Job } from "bullmq";
|
||||
import { queueLatestVideos } from "mq/task/queueLatestVideo.ts";
|
||||
|
||||
export const getLatestVideosWorker = async (_job: Job): Promise<void> =>{
|
||||
export const getLatestVideosWorker = async (_job: Job): Promise<void> => {
|
||||
await queueLatestVideos(sql);
|
||||
}
|
||||
};
|
||||
|
@ -10,4 +10,4 @@ export const getVideoInfoWorker = async (job: Job): Promise<void> => {
|
||||
return;
|
||||
}
|
||||
await insertVideoInfo(sql, aid);
|
||||
}
|
||||
};
|
||||
|
@ -5,15 +5,15 @@ import {
|
||||
getBulkSnapshotsInNextSecond,
|
||||
getSnapshotsInNextSecond,
|
||||
setSnapshotStatus,
|
||||
videoHasProcessingSchedule,
|
||||
videoHasProcessingSchedule
|
||||
} from "db/snapshotSchedule.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
import { SnapshotQueue } from "mq/index.ts";
|
||||
import { sql } from "@core/db/dbNew";
|
||||
|
||||
const priorityMap: { [key: string]: number } = {
|
||||
"milestone": 1,
|
||||
"normal": 3,
|
||||
milestone: 1,
|
||||
normal: 3
|
||||
};
|
||||
|
||||
export const bulkSnapshotTickWorker = async (_job: Job) => {
|
||||
@ -35,12 +35,16 @@ export const bulkSnapshotTickWorker = async (_job: Job) => {
|
||||
created_at: schedule.created_at,
|
||||
started_at: schedule.started_at,
|
||||
finished_at: schedule.finished_at,
|
||||
status: schedule.status,
|
||||
status: schedule.status
|
||||
};
|
||||
});
|
||||
await SnapshotQueue.add("bulkSnapshotVideo", {
|
||||
schedules: schedulesData,
|
||||
}, { priority: 3 });
|
||||
await SnapshotQueue.add(
|
||||
"bulkSnapshotVideo",
|
||||
{
|
||||
schedules: schedulesData
|
||||
},
|
||||
{ priority: 3 }
|
||||
);
|
||||
}
|
||||
return `OK`;
|
||||
} catch (e) {
|
||||
@ -61,11 +65,15 @@ export const snapshotTickWorker = async (_job: Job) => {
|
||||
}
|
||||
const aid = Number(schedule.aid);
|
||||
await setSnapshotStatus(sql, schedule.id, "processing");
|
||||
await SnapshotQueue.add("snapshotVideo", {
|
||||
aid: Number(aid),
|
||||
id: Number(schedule.id),
|
||||
type: schedule.type ?? "normal",
|
||||
}, { priority });
|
||||
await SnapshotQueue.add(
|
||||
"snapshotVideo",
|
||||
{
|
||||
aid: Number(aid),
|
||||
id: Number(schedule.id),
|
||||
type: schedule.type ?? "normal"
|
||||
},
|
||||
{ priority }
|
||||
);
|
||||
}
|
||||
return `OK`;
|
||||
} catch (e) {
|
||||
@ -76,5 +84,5 @@ export const snapshotTickWorker = async (_job: Job) => {
|
||||
export const closetMilestone = (views: number) => {
|
||||
if (views < 100000) return 100000;
|
||||
if (views < 1000000) return 1000000;
|
||||
return 10000000;
|
||||
return Math.ceil(views / 1000000) * 1000000;
|
||||
};
|
||||
|
@ -1,19 +1,19 @@
|
||||
import { Job } from "bullmq";
|
||||
import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
|
||||
import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
import { HOUR, MINUTE, SECOND } from "@core/const/time.ts";
|
||||
import { lockManager } from "@core/mq/lockManager.ts";
|
||||
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts";
|
||||
import { insertVideoSnapshot } from "mq/task/getVideoStats.ts";
|
||||
import { getSongsPublihsedAt } from "db/songs.ts";
|
||||
import { getAdjustedShortTermETA } from "mq/scheduling.ts";
|
||||
import { NetSchedulerError } from "@core/net/delegate.ts";
|
||||
import { sql } from "@core/db/dbNew.ts";
|
||||
import { closetMilestone } from "./snapshotTick.ts";
|
||||
|
||||
const snapshotTypeToTaskMap: { [key: string]: string } = {
|
||||
"milestone": "snapshotMilestoneVideo",
|
||||
"normal": "snapshotVideo",
|
||||
"new": "snapshotMilestoneVideo",
|
||||
milestone: "snapshotMilestoneVideo",
|
||||
normal: "snapshotVideo",
|
||||
new: "snapshotMilestoneVideo"
|
||||
};
|
||||
|
||||
export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
@ -22,6 +22,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
const type = job.data.type;
|
||||
const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo";
|
||||
const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE;
|
||||
const latestSnapshot = await getLatestSnapshot(sql, aid);
|
||||
try {
|
||||
const exists = await snapshotScheduleExists(sql, id);
|
||||
if (!exists) {
|
||||
@ -32,7 +33,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
logger.warn(
|
||||
`Video ${aid} has status ${status} in the database. Abort snapshoting.`,
|
||||
"mq",
|
||||
"fn:dispatchRegularSnapshotsWorker",
|
||||
"fn:dispatchRegularSnapshotsWorker"
|
||||
);
|
||||
return;
|
||||
}
|
||||
@ -44,7 +45,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
logger.warn(
|
||||
`Bilibili return status ${status} when snapshoting for ${aid}.`,
|
||||
"mq",
|
||||
"fn:dispatchRegularSnapshotsWorker",
|
||||
"fn:dispatchRegularSnapshotsWorker"
|
||||
);
|
||||
return;
|
||||
}
|
||||
@ -52,7 +53,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
if (type === "new") {
|
||||
const publihsedAt = await getSongsPublihsedAt(sql, aid);
|
||||
const timeSincePublished = stat.time - publihsedAt!;
|
||||
const viewsPerHour = stat.views / timeSincePublished * HOUR;
|
||||
const viewsPerHour = (stat.views / timeSincePublished) * HOUR;
|
||||
if (timeSincePublished > 48 * HOUR) {
|
||||
return;
|
||||
}
|
||||
@ -72,46 +73,41 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true);
|
||||
}
|
||||
if (type !== "milestone") return;
|
||||
const alreadyAchievedMilestone = stat.views > closetMilestone(latestSnapshot.views);
|
||||
if (alreadyAchievedMilestone) {
|
||||
return;
|
||||
}
|
||||
const eta = await getAdjustedShortTermETA(sql, aid);
|
||||
if (eta > 144) {
|
||||
const etaHoursString = eta.toFixed(2) + " hrs";
|
||||
logger.warn(
|
||||
`ETA (${etaHoursString}) too long for milestone snapshot. aid: ${aid}.`,
|
||||
"mq",
|
||||
"fn:dispatchRegularSnapshotsWorker",
|
||||
"fn:snapshotVideoWorker"
|
||||
);
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const targetTime = now + eta * HOUR;
|
||||
await scheduleSnapshot(sql, aid, type, targetTime);
|
||||
await setSnapshotStatus(sql, id, "completed");
|
||||
return;
|
||||
}
|
||||
catch (e) {
|
||||
} catch (e) {
|
||||
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
|
||||
logger.warn(
|
||||
`No available proxy for aid ${job.data.aid}.`,
|
||||
"mq",
|
||||
"fn:takeSnapshotForVideoWorker",
|
||||
);
|
||||
logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker");
|
||||
await setSnapshotStatus(sql, id, "no_proxy");
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval);
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true);
|
||||
return;
|
||||
}
|
||||
else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
|
||||
} else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") {
|
||||
logger.warn(
|
||||
`Failed to proxy request for aid ${job.data.aid}: ${e.message}`,
|
||||
"mq",
|
||||
"fn:takeSnapshotForVideoWorker",
|
||||
"fn:snapshotVideoWorker"
|
||||
);
|
||||
await setSnapshotStatus(sql, id, "failed");
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval);
|
||||
}
|
||||
logger.error(e as Error, "mq", "fn:takeSnapshotForVideoWorker");
|
||||
logger.error(e as Error, "mq", "fn:snapshotVideoWorker");
|
||||
await setSnapshotStatus(sql, id, "failed");
|
||||
}
|
||||
finally {
|
||||
await lockManager.releaseLock("dispatchRegularSnapshots");
|
||||
};
|
||||
return;
|
||||
};
|
||||
|
@ -3,7 +3,7 @@ import {
|
||||
bulkScheduleSnapshot,
|
||||
bulkSetSnapshotStatus,
|
||||
scheduleSnapshot,
|
||||
snapshotScheduleExists,
|
||||
snapshotScheduleExists
|
||||
} from "db/snapshotSchedule.ts";
|
||||
import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
@ -46,16 +46,16 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
|
||||
await sql`
|
||||
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
|
||||
VALUES (
|
||||
${aid},
|
||||
${views},
|
||||
${danmakus},
|
||||
${replies},
|
||||
${likes},
|
||||
${coins},
|
||||
${shares},
|
||||
${aid},
|
||||
${views},
|
||||
${danmakus},
|
||||
${replies},
|
||||
${likes},
|
||||
${coins},
|
||||
${shares},
|
||||
${favorites}
|
||||
)
|
||||
`
|
||||
`;
|
||||
|
||||
logger.log(`Taken snapshot for video ${aid} in bulk.`, "net", "fn:takeBulkSnapshotForVideosWorker");
|
||||
}
|
||||
@ -72,13 +72,16 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
|
||||
return `DONE`;
|
||||
} catch (e) {
|
||||
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
|
||||
logger.warn(
|
||||
`No available proxy for bulk request now.`,
|
||||
"mq",
|
||||
"fn:takeBulkSnapshotForVideosWorker",
|
||||
);
|
||||
logger.warn(`No available proxy for bulk request now.`, "mq", "fn:takeBulkSnapshotForVideosWorker");
|
||||
await bulkSetSnapshotStatus(sql, ids, "no_proxy");
|
||||
await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random());
|
||||
await bulkScheduleSnapshot(
|
||||
sql,
|
||||
aidsToFetch,
|
||||
"normal",
|
||||
Date.now() + 20 * MINUTE * Math.random(),
|
||||
false,
|
||||
true
|
||||
);
|
||||
return;
|
||||
}
|
||||
logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");
|
||||
|
@ -2,13 +2,13 @@ import { Queue, ConnectionOptions } from "bullmq";
|
||||
import { redis } from "@core/db/redis.ts";
|
||||
|
||||
export const LatestVideosQueue = new Queue("latestVideos", {
|
||||
connection: redis as ConnectionOptions
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
||||
export const ClassifyVideoQueue = new Queue("classifyVideo", {
|
||||
connection: redis as ConnectionOptions
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
||||
export const SnapshotQueue = new Queue("snapshot", {
|
||||
connection: redis as ConnectionOptions
|
||||
connection: redis as ConnectionOptions
|
||||
});
|
||||
|
@ -62,8 +62,8 @@ export async function initMQ() {
|
||||
});
|
||||
|
||||
await SnapshotQueue.upsertJobScheduler("dispatchArchiveSnapshots", {
|
||||
every: 6 * HOUR,
|
||||
immediately: true
|
||||
every: 2 * HOUR,
|
||||
immediately: false
|
||||
});
|
||||
|
||||
await SnapshotQueue.upsertJobScheduler("scheduleCleanup", {
|
||||
|
@ -33,7 +33,7 @@ export const getAdjustedShortTermETA = async (sql: Psql, aid: number) => {
|
||||
if (!snapshotsEnough) return 0;
|
||||
|
||||
const currentTimestamp = new Date().getTime();
|
||||
const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
|
||||
const timeIntervals = [3 * MINUTE, 20 * MINUTE, HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
|
||||
const DELTA = 0.00001;
|
||||
let minETAHours = Infinity;
|
||||
|
||||
|
@ -25,5 +25,5 @@ export async function insertIntoSongs(sql: Psql, aid: number) {
|
||||
(SELECT duration FROM bilibili_metadata WHERE aid = ${aid})
|
||||
)
|
||||
ON CONFLICT DO NOTHING
|
||||
`
|
||||
`;
|
||||
}
|
||||
|
@ -18,9 +18,9 @@ export async function insertVideoInfo(sql: Psql, aid: number) {
|
||||
const bvid = data.View.bvid;
|
||||
const desc = data.View.desc;
|
||||
const uid = data.View.owner.mid;
|
||||
const tags = data.Tags
|
||||
.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type))
|
||||
.map((tag) => tag.tag_name).join(",");
|
||||
const tags = data.Tags.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type))
|
||||
.map((tag) => tag.tag_name)
|
||||
.join(",");
|
||||
const title = data.View.title;
|
||||
const published_at = formatTimestampToPsql(data.View.pubdate * SECOND + 8 * HOUR);
|
||||
const duration = data.View.duration;
|
||||
@ -55,7 +55,7 @@ export async function insertVideoInfo(sql: Psql, aid: number) {
|
||||
${stat.share},
|
||||
${stat.favorite}
|
||||
)
|
||||
`
|
||||
`;
|
||||
|
||||
logger.log(`Inserted video metadata for aid: ${aid}`, "mq");
|
||||
await ClassifyVideoQueue.add("classifyVideo", { aid });
|
||||
|
@ -24,11 +24,7 @@ export interface SnapshotNumber {
|
||||
* - The native `fetch` function threw an error: with error code `FETCH_ERROR`
|
||||
* - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
|
||||
*/
|
||||
export async function insertVideoSnapshot(
|
||||
sql: Psql,
|
||||
aid: number,
|
||||
task: string,
|
||||
): Promise<number | SnapshotNumber> {
|
||||
export async function insertVideoSnapshot(sql: Psql, aid: number, task: string): Promise<number | SnapshotNumber> {
|
||||
const data = await getVideoInfo(aid, task);
|
||||
if (typeof data == "number") {
|
||||
return data;
|
||||
@ -42,10 +38,10 @@ export async function insertVideoSnapshot(
|
||||
const shares = data.stat.share;
|
||||
const favorites = data.stat.favorite;
|
||||
|
||||
await sql`
|
||||
await sql`
|
||||
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
|
||||
VALUES (${aid}, ${views}, ${danmakus}, ${replies}, ${likes}, ${coins}, ${shares}, ${favorites})
|
||||
`
|
||||
`;
|
||||
|
||||
logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
|
||||
|
||||
@ -58,6 +54,6 @@ export async function insertVideoSnapshot(
|
||||
coins,
|
||||
shares,
|
||||
favorites,
|
||||
time,
|
||||
time
|
||||
};
|
||||
}
|
||||
|
@ -6,9 +6,7 @@ import logger from "@core/log/logger.ts";
|
||||
import { LatestVideosQueue } from "mq/index.ts";
|
||||
import type { Psql } from "@core/db/psql.d.ts";
|
||||
|
||||
export async function queueLatestVideos(
|
||||
sql: Psql,
|
||||
): Promise<number | null> {
|
||||
export async function queueLatestVideos(sql: Psql): Promise<number | null> {
|
||||
let page = 1;
|
||||
let i = 0;
|
||||
const videosFound = new Set();
|
||||
@ -26,14 +24,18 @@ export async function queueLatestVideos(
|
||||
if (videoExists) {
|
||||
continue;
|
||||
}
|
||||
await LatestVideosQueue.add("getVideoInfo", { aid }, {
|
||||
delay,
|
||||
attempts: 100,
|
||||
backoff: {
|
||||
type: "fixed",
|
||||
delay: SECOND * 5,
|
||||
},
|
||||
});
|
||||
await LatestVideosQueue.add(
|
||||
"getVideoInfo",
|
||||
{ aid },
|
||||
{
|
||||
delay,
|
||||
attempts: 100,
|
||||
backoff: {
|
||||
type: "fixed",
|
||||
delay: SECOND * 5
|
||||
}
|
||||
}
|
||||
);
|
||||
videosFound.add(aid);
|
||||
allExists = false;
|
||||
delay += Math.random() * SECOND * 1.5;
|
||||
@ -42,7 +44,7 @@ export async function queueLatestVideos(
|
||||
logger.log(
|
||||
`Page ${page} crawled, total: ${videosFound.size}/${i} videos added/observed.`,
|
||||
"net",
|
||||
"fn:queueLatestVideos()",
|
||||
"fn:queueLatestVideos()"
|
||||
);
|
||||
if (allExists) {
|
||||
return 0;
|
||||
|
@ -1,6 +1,6 @@
|
||||
import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts";
|
||||
import { HOUR } from "@core/const/time.ts";
|
||||
import type { Psql } from "@core/db/psql";
|
||||
import type { Psql } from "@core/db/psql.d.ts";
|
||||
|
||||
export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => {
|
||||
const now = Date.now();
|
||||
|
@ -2,14 +2,10 @@ import { sql } from "@core/db/dbNew";
|
||||
import logger from "@core/log/logger.ts";
|
||||
|
||||
export async function removeAllTimeoutSchedules() {
|
||||
logger.log(
|
||||
"Too many timeout schedules, directly removing these schedules...",
|
||||
"mq",
|
||||
"fn:scheduleCleanupWorker",
|
||||
);
|
||||
logger.log("Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker");
|
||||
return await sql`
|
||||
DELETE FROM snapshot_schedule
|
||||
WHERE status IN ('pending', 'processing')
|
||||
AND started_at < NOW() - INTERVAL '30 minutes'
|
||||
`;
|
||||
}
|
||||
}
|
||||
|
@ -1,5 +1,6 @@
|
||||
{
|
||||
"name": "crawler",
|
||||
"version": "1.3.0",
|
||||
"scripts": {
|
||||
"test": "bun --env-file=.env.test run vitest",
|
||||
"worker:main": "bun run ./src/worker.ts",
|
||||
@ -7,7 +8,8 @@
|
||||
"worker:filter": "bun run ./build/filterWorker.js",
|
||||
"adder": "bun run ./src/jobAdder.ts",
|
||||
"bullui": "bun run ./src/bullui.ts",
|
||||
"all": "bun run concurrently --restart-tries -1 'bun run worker:main' 'bun run adder' 'bun run bullui' 'bun run worker:filter'"
|
||||
"all": "bun run concurrently --restart-tries -1 'bun run worker:main' 'bun run adder' 'bun run worker:filter'",
|
||||
"format": "prettier --write ."
|
||||
},
|
||||
"devDependencies": {
|
||||
"concurrently": "^9.1.2"
|
||||
|
@ -3,13 +3,12 @@ import Bun from "bun";
|
||||
await Bun.build({
|
||||
entrypoints: ["./src/filterWorker.ts"],
|
||||
outdir: "./build",
|
||||
target: "node"
|
||||
target: "node"
|
||||
});
|
||||
|
||||
|
||||
const file = Bun.file("./build/filterWorker.js");
|
||||
const code = await file.text();
|
||||
|
||||
const modifiedCode = code.replaceAll("../bin/napi-v3/", "../../../node_modules/onnxruntime-node/bin/napi-v3/");
|
||||
|
||||
await Bun.write("./build/filterWorker.js", modifiedCode);
|
||||
await Bun.write("./build/filterWorker.js", modifiedCode);
|
||||
|
@ -11,9 +11,9 @@ createBullBoard({
|
||||
queues: [
|
||||
new BullMQAdapter(LatestVideosQueue),
|
||||
new BullMQAdapter(ClassifyVideoQueue),
|
||||
new BullMQAdapter(SnapshotQueue),
|
||||
new BullMQAdapter(SnapshotQueue)
|
||||
],
|
||||
serverAdapter: serverAdapter,
|
||||
serverAdapter: serverAdapter
|
||||
});
|
||||
|
||||
const app = express();
|
||||
|
@ -7,13 +7,13 @@ import { lockManager } from "@core/mq/lockManager.ts";
|
||||
import Akari from "ml/akari.ts";
|
||||
|
||||
const shutdown = async (signal: string) => {
|
||||
logger.log(`${signal} Received: Shutting down workers...`, "mq");
|
||||
await filterWorker.close(true);
|
||||
process.exit(0);
|
||||
logger.log(`${signal} Received: Shutting down workers...`, "mq");
|
||||
await filterWorker.close(true);
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
|
||||
await Akari.init();
|
||||
|
||||
@ -29,7 +29,7 @@ const filterWorker = new Worker(
|
||||
break;
|
||||
}
|
||||
},
|
||||
{ connection: redis as ConnectionOptions, concurrency: 2, removeOnComplete: { count: 1000 } },
|
||||
{ connection: redis as ConnectionOptions, concurrency: 2, removeOnComplete: { count: 1000 } }
|
||||
);
|
||||
|
||||
filterWorker.on("active", () => {
|
||||
|
@ -10,7 +10,7 @@ import {
|
||||
scheduleCleanupWorker,
|
||||
snapshotTickWorker,
|
||||
snapshotVideoWorker,
|
||||
takeBulkSnapshotForVideosWorker,
|
||||
takeBulkSnapshotForVideosWorker
|
||||
} from "mq/exec/executors.ts";
|
||||
import { redis } from "@core/db/redis.ts";
|
||||
import logger from "@core/log/logger.ts";
|
||||
@ -30,15 +30,15 @@ const releaseAllLocks = async () => {
|
||||
};
|
||||
|
||||
const shutdown = async (signal: string) => {
|
||||
logger.log(`${signal} Received: Shutting down workers...`, "mq");
|
||||
await releaseAllLocks();
|
||||
logger.log(`${signal} Received: Shutting down workers...`, "mq");
|
||||
await releaseAllLocks();
|
||||
await latestVideoWorker.close(true);
|
||||
await snapshotWorker.close(true);
|
||||
process.exit(0);
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on("SIGINT", () => shutdown("SIGINT"));
|
||||
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
||||
|
||||
const latestVideoWorker = new Worker(
|
||||
"latestVideos",
|
||||
@ -58,8 +58,8 @@ const latestVideoWorker = new Worker(
|
||||
connection: redis as ConnectionOptions,
|
||||
concurrency: 6,
|
||||
removeOnComplete: { count: 1440 },
|
||||
removeOnFail: { count: 0 },
|
||||
},
|
||||
removeOnFail: { count: 0 }
|
||||
}
|
||||
);
|
||||
|
||||
latestVideoWorker.on("active", () => {
|
||||
@ -95,7 +95,7 @@ const snapshotWorker = new Worker(
|
||||
break;
|
||||
}
|
||||
},
|
||||
{ connection: redis as ConnectionOptions, concurrency: 50, removeOnComplete: { count: 2000 } },
|
||||
{ connection: redis as ConnectionOptions, concurrency: 50, removeOnComplete: { count: 2000 } }
|
||||
);
|
||||
|
||||
snapshotWorker.on("error", (err) => {
|
||||
|
@ -56,65 +56,65 @@ const databasePreparationQuery = `
|
||||
CREATE INDEX idx_snapshot_schedule_status ON snapshot_schedule USING btree (status);
|
||||
CREATE INDEX idx_snapshot_schedule_type ON snapshot_schedule USING btree (type);
|
||||
CREATE UNIQUE INDEX snapshot_schedule_pkey ON snapshot_schedule USING btree (id);
|
||||
`
|
||||
`;
|
||||
|
||||
const cleanUpQuery = `
|
||||
DROP SEQUENCE IF EXISTS "snapshot_schedule_id_seq" CASCADE;
|
||||
DROP TABLE IF EXISTS "snapshot_schedule" CASCADE;
|
||||
`
|
||||
`;
|
||||
|
||||
async function testMocking() {
|
||||
await sql.begin(async tx => {
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
await sql.begin(async (tx) => {
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
await tx.unsafe(databasePreparationQuery).simple();
|
||||
|
||||
await tx`
|
||||
|
||||
await tx`
|
||||
INSERT INTO snapshot_schedule
|
||||
${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')}
|
||||
${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
|
||||
`;
|
||||
|
||||
await tx`
|
||||
await tx`
|
||||
ROLLBACK;
|
||||
`
|
||||
`;
|
||||
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
return;
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
return;
|
||||
});
|
||||
}
|
||||
|
||||
async function testBulkSetSnapshotStatus() {
|
||||
return await sql.begin(async tx => {
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
await tx.unsafe(databasePreparationQuery).simple();
|
||||
return await sql.begin(async (tx) => {
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
await tx.unsafe(databasePreparationQuery).simple();
|
||||
|
||||
await tx`
|
||||
await tx`
|
||||
INSERT INTO snapshot_schedule
|
||||
${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')}
|
||||
${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")}
|
||||
`;
|
||||
|
||||
const ids = [1, 2, 3];
|
||||
|
||||
await bulkSetSnapshotStatus(tx, ids, 'pending')
|
||||
|
||||
const rows = tx<{status: string}[]>`
|
||||
await bulkSetSnapshotStatus(tx, ids, "pending");
|
||||
|
||||
const rows = tx<{ status: string }[]>`
|
||||
SELECT status FROM snapshot_schedule WHERE id = 1;
|
||||
`.execute();
|
||||
|
||||
await tx`
|
||||
await tx`
|
||||
ROLLBACK;
|
||||
`
|
||||
`;
|
||||
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
return rows;
|
||||
await tx.unsafe(cleanUpQuery).simple();
|
||||
return rows;
|
||||
});
|
||||
}
|
||||
|
||||
test("data mocking works", async () => {
|
||||
await testMocking();
|
||||
await testMocking();
|
||||
expect(() => {}).not.toThrowError();
|
||||
});
|
||||
|
||||
test("bulkSetSnapshotStatus core logic works smoothly", async () => {
|
||||
const rows = await testBulkSetSnapshotStatus();
|
||||
expect(rows.every(item => item.status === 'pending')).toBe(true);
|
||||
const rows = await testBulkSetSnapshotStatus();
|
||||
expect(rows.every((item) => item.status === "pending")).toBe(true);
|
||||
});
|
||||
|
@ -1,6 +1,6 @@
|
||||
export function formatTimestampToPsql(timestamp: number) {
|
||||
const date = new Date(timestamp);
|
||||
return date.toISOString().slice(0, 23).replace("T", " ");
|
||||
return date.toISOString().slice(0, 23).replace("T", " ") + "+08";
|
||||
}
|
||||
|
||||
export function parseTimestampFromPsql(timestamp: string) {
|
||||
|
@ -1,6 +1,6 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
import tsconfigPaths from 'vite-tsconfig-paths'
|
||||
import tsconfigPaths from "vite-tsconfig-paths";
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [tsconfigPaths()]
|
||||
plugins: [tsconfigPaths()]
|
||||
});
|
||||
|
@ -1,31 +1,31 @@
|
||||
{
|
||||
"name": "frontend",
|
||||
"type": "module",
|
||||
"version": "0.0.1",
|
||||
"scripts": {
|
||||
"dev": "astro dev",
|
||||
"build": "astro build",
|
||||
"preview": "astro preview",
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/node": "^9.1.3",
|
||||
"@astrojs/svelte": "^7.0.9",
|
||||
"@tailwindcss/vite": "^4.1.4",
|
||||
"argon2id": "^1.0.1",
|
||||
"astro": "^5.5.5",
|
||||
"autoprefixer": "^10.4.21",
|
||||
"date-fns": "^4.1.0",
|
||||
"pg": "^8.11.11",
|
||||
"postcss": "^8.5.3",
|
||||
"postgres": "^3.4.5",
|
||||
"svelte": "^5.25.7",
|
||||
"tailwindcss": "^4.1.4",
|
||||
"ua-parser-js": "^2.0.3",
|
||||
"vite-tsconfig-paths": "^5.1.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-wasm": "^6.2.2",
|
||||
"@types/pg": "^8.11.11"
|
||||
}
|
||||
"name": "frontend",
|
||||
"type": "module",
|
||||
"version": "1.9.0",
|
||||
"scripts": {
|
||||
"dev": "astro dev",
|
||||
"build": "astro build",
|
||||
"preview": "astro preview",
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/node": "^9.1.3",
|
||||
"@astrojs/svelte": "^7.0.9",
|
||||
"@tailwindcss/vite": "^4.1.4",
|
||||
"argon2id": "^1.0.1",
|
||||
"astro": "^5.5.5",
|
||||
"autoprefixer": "^10.4.21",
|
||||
"date-fns": "^4.1.0",
|
||||
"pg": "^8.11.11",
|
||||
"postcss": "^8.5.3",
|
||||
"postgres": "^3.4.5",
|
||||
"svelte": "^5.25.7",
|
||||
"tailwindcss": "^4.1.4",
|
||||
"ua-parser-js": "^2.0.3",
|
||||
"vite-tsconfig-paths": "^5.1.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@rollup/plugin-wasm": "^6.2.2",
|
||||
"@types/pg": "^8.11.11"
|
||||
}
|
||||
}
|
||||
|
@ -1,9 +1,20 @@
|
||||
import { LeftArrow } from "@/components/icons/LeftArrow";
|
||||
import { RightArrow } from "@/components/icons/RightArrow";
|
||||
import LoginForm from "./LoginForm";
|
||||
import { Link } from "@/i18n/navigation";
|
||||
import { Link, redirect } from "@/i18n/navigation";
|
||||
import { getLocale } from "next-intl/server";
|
||||
import { getCurrentUser } from "@/lib/userAuth";
|
||||
|
||||
export default function SignupPage() {
|
||||
export default async function LoginPage() {
|
||||
const user = await getCurrentUser();
|
||||
const locale = await getLocale();
|
||||
|
||||
if (user) {
|
||||
redirect({
|
||||
href: `/user/${user.uid}/profile`,
|
||||
locale: locale
|
||||
});
|
||||
}
|
||||
return (
|
||||
<main className="relative flex-grow pt-8 px-4 md:w-full md:h-full md:flex md:items-center md:justify-center">
|
||||
<div
|
||||
|
60
packages/next/app/[locale]/logout/route.ts
Normal file
60
packages/next/app/[locale]/logout/route.ts
Normal file
@ -0,0 +1,60 @@
|
||||
import { ApiRequestError, fetcher } from "@/lib/net";
|
||||
import { ErrorResponse } from "@cvsa/backend";
|
||||
import { cookies } from "next/headers";
|
||||
|
||||
export async function POST() {
|
||||
const backendURL = process.env.BACKEND_URL || "";
|
||||
const cookieStore = await cookies();
|
||||
const sessionID = cookieStore.get("session_id");
|
||||
if (!sessionID) {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "No session_id provided",
|
||||
errors: [],
|
||||
code: "ENTITY_NOT_FOUND"
|
||||
};
|
||||
return new Response(JSON.stringify(response), {
|
||||
status: 401
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetcher(`${backendURL}/session/${sessionID.value}`, {
|
||||
method: "DELETE"
|
||||
});
|
||||
|
||||
const headers = response.headers;
|
||||
|
||||
return new Response(null, {
|
||||
status: 204,
|
||||
headers: {
|
||||
"Set-Cookie": (headers["set-cookie"] || [""])[0]
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof ApiRequestError && error.response) {
|
||||
const res = error.response;
|
||||
const code = error.code;
|
||||
return new Response(JSON.stringify(res), {
|
||||
status: code
|
||||
});
|
||||
} else if (error instanceof Error) {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: error.message,
|
||||
errors: [],
|
||||
code: "SERVER_ERROR"
|
||||
};
|
||||
return new Response(JSON.stringify(response), {
|
||||
status: 500
|
||||
});
|
||||
} else {
|
||||
const response: ErrorResponse<string> = {
|
||||
message: "Unknown error occurred",
|
||||
errors: [],
|
||||
code: "UNKNOWN_ERROR"
|
||||
};
|
||||
return new Response(JSON.stringify(response), {
|
||||
status: 500
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
@ -1,46 +0,0 @@
|
||||
import { Header } from "@/components/shell/Header";
|
||||
import { getCurrentUser, User } from "@/lib/userAuth";
|
||||
import { redirect } from "next/navigation";
|
||||
import { format } from "date-fns";
|
||||
import { zhCN } from "date-fns/locale";
|
||||
|
||||
interface SignupTimeProps {
|
||||
user: User;
|
||||
}
|
||||
|
||||
const SignupTime: React.FC<SignupTimeProps> = ({ user }: SignupTimeProps) => {
|
||||
return (
|
||||
<p className="mt-4">
|
||||
于
|
||||
{format(new Date(user.createdAt), "yyyy-MM-dd HH:mm:ss", {
|
||||
locale: zhCN
|
||||
})}
|
||||
注册。
|
||||
</p>
|
||||
);
|
||||
};
|
||||
|
||||
export default async function ProfilePage() {
|
||||
const user = await getCurrentUser();
|
||||
|
||||
if (!user) {
|
||||
redirect("/login");
|
||||
}
|
||||
|
||||
const displayName = user.nickname || user.username;
|
||||
|
||||
return (
|
||||
<>
|
||||
<Header user={user} />
|
||||
<main className="md:w-xl lg:w-2xl xl:w-3xl md:mx-auto pt-6">
|
||||
<h1>
|
||||
<span className="text-4xl font-extralight">{displayName}</span>
|
||||
<span className="ml-2 text-on-surface-variant dark:text-dark-on-surface-variant">
|
||||
UID{user.uid}
|
||||
</span>
|
||||
</h1>
|
||||
<SignupTime user={user} />
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
@ -7,6 +7,7 @@ import { aidExists as idExists } from "@/lib/db/bilibili_metadata/aidExists";
|
||||
import { notFound } from "next/navigation";
|
||||
import { BiliVideoMetadataType, VideoSnapshotType } from "@cvsa/core";
|
||||
import { Metadata } from "next";
|
||||
import { DateTime } from "luxon";
|
||||
|
||||
const MetadataRow = ({ title, desc }: { title: string; desc: string | number | undefined | null }) => {
|
||||
if (!desc) return <></>;
|
||||
@ -104,18 +105,16 @@ export default async function VideoInfoPage({ params }: { params: Promise<{ id:
|
||||
title="发布时间"
|
||||
desc={
|
||||
videoInfo.published_at
|
||||
? format(new Date(videoInfo.published_at), "yyyy-MM-dd HH:mm:ss", {
|
||||
locale: zhCN
|
||||
})
|
||||
? DateTime.fromJSDate(videoInfo.published_at).toFormat(
|
||||
"yyyy-MM-dd HH:mm:ss"
|
||||
)
|
||||
: null
|
||||
}
|
||||
/>
|
||||
<MetadataRow title="时长 (秒)" desc={videoInfo.duration} />
|
||||
<MetadataRow
|
||||
title="创建时间"
|
||||
desc={format(new Date(videoInfo.created_at), "yyyy-MM-dd HH:mm:ss", {
|
||||
locale: zhCN
|
||||
})}
|
||||
desc={DateTime.fromJSDate(videoInfo.created_at).toFormat("yyyy-MM-dd HH:mm:ss")}
|
||||
/>
|
||||
<MetadataRow title="封面" desc={videoInfo?.cover_url} />
|
||||
</tbody>
|
||||
@ -142,11 +141,11 @@ export default async function VideoInfoPage({ params }: { params: Promise<{ id:
|
||||
</thead>
|
||||
<tbody>
|
||||
{snapshots.map((snapshot) => (
|
||||
<tr key={snapshot.created_at}>
|
||||
<tr key={snapshot.id}>
|
||||
<td className="border dark:border-zinc-500 px-4 py-2">
|
||||
{format(new Date(snapshot.created_at), "yyyy-MM-dd HH:mm:ss", {
|
||||
locale: zhCN
|
||||
})}
|
||||
{DateTime.fromJSDate(snapshot.created_at).toFormat(
|
||||
"yyyy-MM-dd HH:mm:ss"
|
||||
)}
|
||||
</td>
|
||||
<td className="border dark:border-zinc-500 px-4 py-2">{snapshot.views}</td>
|
||||
<td className="border dark:border-zinc-500 px-4 py-2">{snapshot.coins}</td>
|
||||
|
@ -1,5 +1,4 @@
|
||||
import { Header } from "@/components/shell/Header";
|
||||
import { getCurrentUser } from "@/lib/userAuth";
|
||||
import HeaderServer from "@/components/shell/HeaderServer";
|
||||
import React from "react";
|
||||
|
||||
export default async function RootLayout({
|
||||
@ -7,10 +6,9 @@ export default async function RootLayout({
|
||||
}: Readonly<{
|
||||
children: React.ReactNode;
|
||||
}>) {
|
||||
const user = await getCurrentUser();
|
||||
return (
|
||||
<>
|
||||
<Header user={user} />
|
||||
<HeaderServer />
|
||||
{children}
|
||||
</>
|
||||
);
|
||||
|
@ -1,13 +1,10 @@
|
||||
import { Header } from "@/components/shell/Header";
|
||||
import { getCurrentUser } from "@/lib/userAuth";
|
||||
import { VDFtestCard } from "./TestCard";
|
||||
import HeaderServer from "@/components/shell/HeaderServer";
|
||||
|
||||
export default async function VdfBenchmarkPage() {
|
||||
const user = await getCurrentUser();
|
||||
|
||||
return (
|
||||
<>
|
||||
<Header user={user} />
|
||||
<HeaderServer />
|
||||
<div className="md:w-2/3 lg:w-1/2 xl:w-[37%] md:mx-auto mx-6 mb-12">
|
||||
<VDFtestCard />
|
||||
<div>
|
||||
|
@ -0,0 +1,46 @@
|
||||
"use client";
|
||||
|
||||
import { FilledButton } from "@/components/ui/Buttons/FilledButton";
|
||||
import { Dialog, DialogButton, DialogButtonGroup, DialogHeadline, DialogSupportingText } from "@/components/ui/Dialog";
|
||||
import { Portal } from "@/components/utils/Portal";
|
||||
import { useRouter } from "@/i18n/navigation";
|
||||
import { useState } from "react";
|
||||
|
||||
export const LogoutButton: React.FC = () => {
|
||||
const [showDialog, setShowDialog] = useState(false);
|
||||
const router = useRouter();
|
||||
return (
|
||||
<>
|
||||
<FilledButton
|
||||
shape="square"
|
||||
className="mt-5 !text-on-error dark:!text-dark-on-error !bg-error dark:!bg-dark-error font-medium"
|
||||
onClick={() => setShowDialog(true)}
|
||||
>
|
||||
登出
|
||||
</FilledButton>
|
||||
<Portal>
|
||||
<Dialog show={showDialog}>
|
||||
<DialogHeadline>确认登出</DialogHeadline>
|
||||
<DialogSupportingText>确认要退出登录吗?</DialogSupportingText>
|
||||
<DialogButtonGroup close={() => setShowDialog(false)}>
|
||||
<DialogButton onClick={() => setShowDialog(false)}>取消</DialogButton>
|
||||
<DialogButton
|
||||
onClick={async () => {
|
||||
try {
|
||||
await fetch("/logout", {
|
||||
method: "POST"
|
||||
});
|
||||
router.push("/");
|
||||
} finally {
|
||||
setShowDialog(false);
|
||||
}
|
||||
}}
|
||||
>
|
||||
确认
|
||||
</DialogButton>
|
||||
</DialogButtonGroup>
|
||||
</Dialog>
|
||||
</Portal>
|
||||
</>
|
||||
);
|
||||
};
|
64
packages/next/app/[locale]/user/[uid]/profile/page.tsx
Normal file
64
packages/next/app/[locale]/user/[uid]/profile/page.tsx
Normal file
@ -0,0 +1,64 @@
|
||||
import { getUserProfile, User } from "@/lib/userAuth";
|
||||
import { notFound } from "next/navigation";
|
||||
import { format } from "date-fns";
|
||||
import { zhCN } from "date-fns/locale";
|
||||
import { LogoutButton } from "./LogoutButton";
|
||||
import { numeric } from "yup-numeric";
|
||||
import { getTranslations } from "next-intl/server";
|
||||
import HeaderServer from "@/components/shell/HeaderServer";
|
||||
import { DateTime } from "luxon";
|
||||
|
||||
const uidSchema = numeric().integer().min(0);
|
||||
|
||||
interface SignupTimeProps {
|
||||
user: User;
|
||||
}
|
||||
|
||||
const SignupTime: React.FC<SignupTimeProps> = ({ user }: SignupTimeProps) => {
|
||||
return (
|
||||
<p className="mt-4">
|
||||
于
|
||||
{DateTime.fromJSDate(user.createdAt).toFormat("yyyy-MM-dd HH:mm:ss")}
|
||||
注册。
|
||||
</p>
|
||||
);
|
||||
};
|
||||
|
||||
export default async function ProfilePage({ params }: { params: Promise<{ uid: string }> }) {
|
||||
const { uid } = await params;
|
||||
const t = await getTranslations("profile_page");
|
||||
let parsedUID: number;
|
||||
|
||||
try {
|
||||
uidSchema.validate(uid);
|
||||
parsedUID = parseInt(uid);
|
||||
} catch (error) {
|
||||
return notFound();
|
||||
}
|
||||
|
||||
const user = await getUserProfile(parsedUID);
|
||||
|
||||
if (!user) {
|
||||
return notFound();
|
||||
}
|
||||
|
||||
const displayName = user.nickname || user.username;
|
||||
const loggedIn = user.isLoggedIn;
|
||||
|
||||
return (
|
||||
<>
|
||||
<HeaderServer />
|
||||
<main className="md:w-xl lg:w-2xl xl:w-3xl md:mx-auto pt-6">
|
||||
<h1>
|
||||
<span className="text-4xl font-extralight">{displayName}</span>
|
||||
<span className="ml-2 text-on-surface-variant dark:text-dark-on-surface-variant">
|
||||
UID{user.uid}
|
||||
</span>
|
||||
</h1>
|
||||
<SignupTime user={user} />
|
||||
<p className="mt-4">权限组:{t(`role.${user.role}`)}</p>
|
||||
{loggedIn && <LogoutButton />}
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
@ -1,9 +1,9 @@
|
||||
import { Suspense } from "react";
|
||||
import Link from "next/link";
|
||||
import { format } from "date-fns";
|
||||
import { notFound } from "next/navigation";
|
||||
import { Metadata } from "next";
|
||||
import type { VideoInfoData } from "@cvsa/core";
|
||||
import { DateTime } from "luxon";
|
||||
|
||||
const StatRow = ({ title, description }: { title: string; description?: number }) => {
|
||||
return (
|
||||
@ -51,7 +51,7 @@ const VideoInfo = async ({ id }: { id: string }) => {
|
||||
{data.bvid} · av{data.aid}
|
||||
</span>
|
||||
<br />
|
||||
<span>发布于 {format(new Date(data.pubdate * 1000), "yyyy-MM-dd HH:mm:ss")}</span>
|
||||
<span>发布于 {DateTime.fromSeconds(data.pubdate).toFormat("yyyy-MM-dd HH:mm:ss")}</span>
|
||||
<br />
|
||||
<span>播放:{(data.stat?.view ?? 0).toLocaleString()}</span> ·{" "}
|
||||
<span>弹幕:{(data.stat?.danmaku ?? 0).toLocaleString()}</span>
|
||||
|
@ -1,5 +1,4 @@
|
||||
import { Header } from "@/components/shell/Header";
|
||||
import { getCurrentUser } from "@/lib/userAuth";
|
||||
import HeaderServer from "@/components/shell/HeaderServer";
|
||||
import React from "react";
|
||||
|
||||
export default async function RootLayout({
|
||||
@ -7,10 +6,9 @@ export default async function RootLayout({
|
||||
}: Readonly<{
|
||||
children: React.ReactNode;
|
||||
}>) {
|
||||
const user = await getCurrentUser();
|
||||
return (
|
||||
<>
|
||||
<Header user={user} />
|
||||
<HeaderServer />
|
||||
{children}
|
||||
</>
|
||||
);
|
||||
|
@ -5,16 +5,19 @@
|
||||
"name": "next",
|
||||
"dependencies": {
|
||||
"@cvsa/backend": "^0.5.3",
|
||||
"@cvsa/core": "^0.0.7",
|
||||
"@cvsa/core": "0.0.10",
|
||||
"@mdx-js/loader": "^3.1.0",
|
||||
"@mdx-js/react": "^3.1.0",
|
||||
"@next/mdx": "^15.3.3",
|
||||
"@types/luxon": "^3.6.2",
|
||||
"@types/mdx": "^2.0.13",
|
||||
"axios": "^1.9.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"framer-motion": "^12.15.0",
|
||||
"fumadocs-mdx": "^11.6.6",
|
||||
"i18next": "^25.2.1",
|
||||
"jotai": "^2.12.5",
|
||||
"luxon": "^3.6.1",
|
||||
"next": "^15.3.3",
|
||||
"next-intl": "^4.1.0",
|
||||
"raw-loader": "^4.0.2",
|
||||
@ -23,6 +26,7 @@
|
||||
"swr": "^2.3.3",
|
||||
"ua-parser-js": "^2.0.3",
|
||||
"yup": "^1.6.1",
|
||||
"yup-numeric": "^0.5.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4.1.8",
|
||||
@ -45,7 +49,7 @@
|
||||
|
||||
"@cvsa/backend": ["@cvsa/backend@0.5.3", "", { "dependencies": { "@rabbit-company/argon2id": "^2.1.0", "hono": "^4.7.8", "hono-rate-limiter": "^0.4.2", "ioredis": "^5.6.1", "limiter": "^3.0.0", "postgres": "^3.4.5", "rate-limit-redis": "^4.2.0", "yup": "^1.6.1", "zod": "^3.24.3" } }, "sha512-RzGjarU2TOzD6/d6qikE4xd/ZqNQl3jOYtgfJg5kbWFuiXnOgEC9QBTi+adzjmaWFrcpuYck6ooWpg4eT3s43g=="],
|
||||
|
||||
"@cvsa/core": ["@cvsa/core@0.0.7", "", { "dependencies": { "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", "postgres": "^3.4.5", "winston": "^3.17.0" } }, "sha512-j2Ksg+ZquHqKPew1JZxw0Q9yckFnzdd8y+DnmVT+OW18j+pKcduB9j0qqBywQGHxGuDYVOGLiPlf+IBXfqQWTg=="],
|
||||
"@cvsa/core": ["@cvsa/core@0.0.10", "", { "dependencies": { "@koshnic/ratelimit": "^1.0.3", "chalk": "^5.4.1", "ioredis": "^5.6.1", "logform": "^2.7.0", "postgres": "^3.4.5", "winston": "^3.17.0" } }, "sha512-8gjSNRyLZcybLiFSUZFPc4nJsLQ7YO8lZSAEFJidyUA3a6CbB/UUC4G5jqWyWJ7xDA39w7szqpbVYKX3fb6W3g=="],
|
||||
|
||||
"@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="],
|
||||
|
||||
@ -273,6 +277,8 @@
|
||||
|
||||
"@types/json-schema": ["@types/json-schema@7.0.15", "", {}, "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA=="],
|
||||
|
||||
"@types/luxon": ["@types/luxon@3.6.2", "", {}, "sha512-R/BdP7OxEMc44l2Ex5lSXHoIXTB2JLNa3y2QISIbr58U/YcsffyQrYW//hZSdrfxrjRZj3GcUoxMPGdO8gSYuw=="],
|
||||
|
||||
"@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="],
|
||||
|
||||
"@types/mdx": ["@types/mdx@2.0.13", "", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="],
|
||||
@ -367,6 +373,8 @@
|
||||
|
||||
"big.js": ["big.js@5.2.2", "", {}, "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ=="],
|
||||
|
||||
"bignumber.js": ["bignumber.js@9.3.0", "", {}, "sha512-EM7aMFTXbptt/wZdMlBv2t8IViwQL+h6SLHosp8Yf0dqJMTnY6iL32opnAB6kAdL0SZPuvcAzFr31o0c/R3/RA=="],
|
||||
|
||||
"binary-extensions": ["binary-extensions@2.3.0", "", {}, "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw=="],
|
||||
|
||||
"body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
|
||||
@ -701,6 +709,8 @@
|
||||
|
||||
"jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="],
|
||||
|
||||
"jotai": ["jotai@2.12.5", "", { "peerDependencies": { "@types/react": ">=17.0.0", "react": ">=17.0.0" }, "optionalPeers": ["@types/react", "react"] }, "sha512-G8m32HW3lSmcz/4mbqx0hgJIQ0ekndKWiYP7kWVKi0p6saLXdSoye+FZiOFyonnd7Q482LCzm8sMDl7Ar1NWDw=="],
|
||||
|
||||
"js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="],
|
||||
|
||||
"json-parse-even-better-errors": ["json-parse-even-better-errors@2.3.1", "", {}, "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w=="],
|
||||
@ -757,6 +767,8 @@
|
||||
|
||||
"lru-cache": ["lru-cache@11.1.0", "", {}, "sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A=="],
|
||||
|
||||
"luxon": ["luxon@3.6.1", "", {}, "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="],
|
||||
|
||||
"magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="],
|
||||
|
||||
"markdown-extensions": ["markdown-extensions@2.0.0", "", {}, "sha512-o5vL7aDWatOTX8LzaS1WMoaoxIiLRQJuIKKe2wAw6IeULDHaqbiqiggmx+pKvZDb1Sj+pE46Sn1T7lCqfFtg1Q=="],
|
||||
@ -1213,6 +1225,8 @@
|
||||
|
||||
"yup": ["yup@1.6.1", "", { "dependencies": { "property-expr": "^2.0.5", "tiny-case": "^1.0.3", "toposort": "^2.0.2", "type-fest": "^2.19.0" } }, "sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA=="],
|
||||
|
||||
"yup-numeric": ["yup-numeric@0.5.0", "", { "dependencies": { "typescript": "^5.4.2" }, "peerDependencies": { "bignumber.js": "^9.1.2", "yup": "^1.3.3" } }, "sha512-IrkLyIY0jLwtomVArrjV1Sv2YHOC715UdRPA7WfAJ0upARXLtmnmzszlPQeEoUxtSb3E9mrF8DoFgiQcRkxOLA=="],
|
||||
|
||||
"zod": ["zod@3.25.46", "", {}, "sha512-IqRxcHEIjqLd4LNS/zKffB3Jzg3NwqJxQQ0Ns7pdrvgGkwQsEBdEQcOHaBVqvvZArShRzI39+aMST3FBGmTrLQ=="],
|
||||
|
||||
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
|
||||
|
@ -15,12 +15,12 @@ import { InfoIcon } from "@/components/icons/InfoIcon";
|
||||
import { HomeIcon } from "@/components/icons/HomeIcon";
|
||||
import { TextButton } from "@/components/ui/Buttons/TextButton";
|
||||
import { Link } from "@/i18n/navigation";
|
||||
import type { UserResponse } from "@cvsa/backend";
|
||||
import { LoginIcon } from "../icons/LoginIcon";
|
||||
import { AccountIcon } from "../icons/AccountIcon";
|
||||
import { User } from "@/lib/userAuth";
|
||||
|
||||
interface HeaderProps {
|
||||
user: UserResponse | null;
|
||||
user: User | null;
|
||||
}
|
||||
|
||||
export const HeaderDestop = ({ user }: HeaderProps) => {
|
||||
@ -44,7 +44,7 @@ export const HeaderDestop = ({ user }: HeaderProps) => {
|
||||
text-xl font-medium items-center w-[15rem] min-w-[8rem] mr-4 lg:mr-0 lg:w-[305px] justify-end"
|
||||
>
|
||||
{user ? (
|
||||
<Link href="/my/profile">{user.nickname || user.username}</Link>
|
||||
<Link href={`/user/${user.uid}/profile`}>{user.nickname || user.username}</Link>
|
||||
) : (
|
||||
<Link href="/login">登录</Link>
|
||||
)}
|
||||
@ -91,7 +91,7 @@ export const HeaderMobile = ({ user }: HeaderProps) => {
|
||||
</Link>
|
||||
|
||||
{user ? (
|
||||
<Link href="/my/profile">
|
||||
<Link href={`/user/${user.uid}/profile`}>
|
||||
<TextButton className="w-full h-14 flex justify-start" size="m">
|
||||
<div className="flex items-center w-72">
|
||||
<AccountIcon className="text-2xl pr-4" />
|
||||
|
@ -1,3 +1,5 @@
|
||||
"use client";
|
||||
|
||||
import useRipple from "@/components/utils/useRipple";
|
||||
|
||||
interface FilledButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement> {
|
||||
|
@ -1,3 +1,5 @@
|
||||
"use client";
|
||||
|
||||
import useRipple from "@/components/utils/useRipple";
|
||||
|
||||
interface TextButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement> {
|
||||
@ -5,13 +7,16 @@ interface TextButtonProps extends React.ButtonHTMLAttributes<HTMLButtonElement>
|
||||
shape?: "round" | "square";
|
||||
children?: React.ReactNode;
|
||||
ripple?: boolean;
|
||||
ref?: React.Ref<HTMLButtonElement>;
|
||||
}
|
||||
|
||||
export const TextButton = ({
|
||||
children,
|
||||
size = "s",
|
||||
shape = "round",
|
||||
className,
|
||||
className = "",
|
||||
disabled,
|
||||
ref,
|
||||
ripple = true,
|
||||
...rest
|
||||
}: TextButtonProps) => {
|
||||
@ -29,12 +34,19 @@ export const TextButton = ({
|
||||
<button
|
||||
className={`text-primary dark:text-dark-primary duration-150 select-none
|
||||
flex items-center justify-center relative overflow-hidden
|
||||
disabled:text-on-surface/40 disabled:dark:text-dark-on-surface/40
|
||||
${sizeClasses} ${shapeClasses} ${className}`}
|
||||
{...rest}
|
||||
onMouseDown={onMouseDown}
|
||||
onTouchStart={onTouchStart}
|
||||
disabled={disabled}
|
||||
ref={ref}
|
||||
>
|
||||
<div className="absolute w-full h-full hover:bg-primary/10 left-0 top-0"></div>
|
||||
<div
|
||||
className={`absolute w-full h-full enabled:hover:bg-primary/10 enabled:dark:hover:bg-dark-primary/10
|
||||
${disabled && "bg-on-surface/10 dark:bg-dark-on-surface/10"}
|
||||
left-0 top-0`}
|
||||
></div>
|
||||
{children}
|
||||
</button>
|
||||
);
|
||||
|
@ -1,7 +1,12 @@
|
||||
import { motion, AnimatePresence } from "framer-motion";
|
||||
import React from "react";
|
||||
import React, { useRef } from "react";
|
||||
import { TextButton } from "./Buttons/TextButton";
|
||||
import { useEffect } from "react";
|
||||
import { useEffect, useState } from "react";
|
||||
import { atom, useAtom, useAtomValue, useSetAtom } from "jotai";
|
||||
import { useKeyboardShortcuts } from "@/components/utils/useKeyboardEvents";
|
||||
import { UAParser } from "ua-parser-js";
|
||||
|
||||
const focusedButtonAtom = atom(-1);
|
||||
|
||||
export const useDisableBodyScroll = (open: boolean) => {
|
||||
useEffect(() => {
|
||||
@ -23,10 +28,14 @@ type ButtonElementAttr = React.HTMLAttributes<HTMLButtonElement>;
|
||||
|
||||
type DialogHeadlineProps = OptionalChidrenProps<HeadElementAttr>;
|
||||
type DialogSupportingTextProps = OptionalChidrenProps<DivElementAttr>;
|
||||
type DialogButtonGroupProps = OptionalChidrenProps<DivElementAttr>;
|
||||
type DialogButtonGroupProps = DivElementAttr & {
|
||||
children: React.ReactElement<DialogButtonProps> | React.ReactElement<DialogButtonProps>[];
|
||||
close: () => void;
|
||||
};
|
||||
|
||||
interface DialogButtonProps extends OptionalChidrenProps<ButtonElementAttr> {
|
||||
onClick?: React.MouseEventHandler<HTMLButtonElement>;
|
||||
index?: number;
|
||||
}
|
||||
interface DialogProps extends OptionalChidrenProps<DivElementAttr> {
|
||||
show: boolean;
|
||||
@ -63,46 +72,178 @@ export const DialogSupportingText: React.FC<DialogSupportingTextProps> = ({
|
||||
);
|
||||
};
|
||||
|
||||
export const DialogButton: React.FC<DialogButtonProps> = ({ children, onClick, ...rest }: DialogButtonProps) => {
|
||||
export const DialogButton: React.FC<DialogButtonProps> = ({ children, onClick, index, ...rest }: DialogButtonProps) => {
|
||||
const buttonRef = useRef<HTMLButtonElement>(null);
|
||||
const focusedButton = useAtomValue(focusedButtonAtom);
|
||||
|
||||
useEffect(() => {
|
||||
if (!buttonRef.current) return;
|
||||
if (focusedButton === index) buttonRef.current.focus();
|
||||
}, [focusedButton]);
|
||||
|
||||
return (
|
||||
<TextButton onClick={onClick} {...rest}>
|
||||
<TextButton onClick={onClick} {...rest} ref={buttonRef}>
|
||||
{children}
|
||||
</TextButton>
|
||||
);
|
||||
};
|
||||
|
||||
export const DialogButtonGroup: React.FC<DialogButtonGroupProps> = ({ children, ...rest }: DialogButtonGroupProps) => {
|
||||
export const DialogButtonGroup: React.FC<DialogButtonGroupProps> = ({
|
||||
children,
|
||||
close,
|
||||
...rest
|
||||
}: DialogButtonGroupProps) => {
|
||||
const [focusedButton, setFocusedButton] = useAtom(focusedButtonAtom);
|
||||
const count = React.Children.count(children);
|
||||
|
||||
useKeyboardShortcuts([
|
||||
{
|
||||
key: "Tab",
|
||||
callback: () => {
|
||||
setFocusedButton((focusedButton + 1) % count);
|
||||
},
|
||||
preventDefault: true
|
||||
},
|
||||
{
|
||||
key: "Escape",
|
||||
callback: close,
|
||||
preventDefault: true
|
||||
}
|
||||
]);
|
||||
|
||||
return (
|
||||
<div className="flex justify-end gap-2" {...rest}>
|
||||
{children}
|
||||
{React.Children.map(children, (child, index) => {
|
||||
if (React.isValidElement<DialogButtonProps>(child) && child.type === DialogButton) {
|
||||
return React.cloneElement(child, {
|
||||
index: index
|
||||
});
|
||||
}
|
||||
return child;
|
||||
})}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
const useCompabilityCheck = () => {
|
||||
const [supported, setSupported] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
const parser = new UAParser(navigator.userAgent);
|
||||
const result = parser.getResult();
|
||||
|
||||
const { name: browserName, version: browserVersion } = result.browser;
|
||||
|
||||
let isSupported = false;
|
||||
|
||||
if (!browserVersion) {
|
||||
return;
|
||||
}
|
||||
const [major] = browserVersion.split(".").map(Number);
|
||||
|
||||
switch (browserName) {
|
||||
case "Chromium":
|
||||
isSupported = major >= 107;
|
||||
break;
|
||||
case "Firefox":
|
||||
isSupported = major >= 66;
|
||||
break;
|
||||
case "Safari":
|
||||
isSupported = major >= 16;
|
||||
break;
|
||||
default:
|
||||
isSupported = false;
|
||||
break;
|
||||
}
|
||||
|
||||
setSupported(isSupported);
|
||||
}, []);
|
||||
|
||||
return supported;
|
||||
};
|
||||
|
||||
export const Dialog: React.FC<DialogProps> = ({ show, children, className }: DialogProps) => {
|
||||
const dialogRef = useRef<HTMLDivElement>(null);
|
||||
const contentRef = useRef<HTMLDivElement>(null);
|
||||
const setFocusedButton = useSetAtom(focusedButtonAtom);
|
||||
const isSupported = useCompabilityCheck();
|
||||
|
||||
useEffect(() => {
|
||||
if (!contentRef.current || !dialogRef.current) return;
|
||||
|
||||
const contentHeight = contentRef.current.offsetHeight;
|
||||
const halfSize = (contentHeight + 48) / 2;
|
||||
dialogRef.current.style.top = `calc(50% - ${halfSize}px)`;
|
||||
|
||||
if (!isSupported) {
|
||||
return;
|
||||
}
|
||||
|
||||
dialogRef.current.style.transition = "grid-template-rows cubic-bezier(0.05, 0.7, 0.1, 1.0) 0.35s";
|
||||
|
||||
if (show) {
|
||||
dialogRef.current.style.gridTemplateRows = "1fr";
|
||||
} else {
|
||||
dialogRef.current.style.gridTemplateRows = "0.6fr";
|
||||
}
|
||||
}, [show]);
|
||||
|
||||
useEffect(() => {
|
||||
setFocusedButton(-1);
|
||||
}, [show]);
|
||||
|
||||
useDisableBodyScroll(show);
|
||||
|
||||
return (
|
||||
<AnimatePresence>
|
||||
{show && (
|
||||
<div className="w-full h-full top-0 left-0 absolute flex items-center justify-center">
|
||||
<div className="w-full h-full top-0 left-0 absolute flex justify-center">
|
||||
<motion.div
|
||||
className="fixed top-0 left-0 w-full h-full z-40 bg-black/20 pointer-none"
|
||||
aria-hidden="true"
|
||||
initial={{ opacity: 0 }}
|
||||
animate={{ opacity: 1 }}
|
||||
exit={{ opacity: 0 }}
|
||||
transition={{ duration: 0.3 }}
|
||||
transition={{ duration: 0.35 }}
|
||||
/>
|
||||
<motion.div
|
||||
className={`fixed min-w-[17.5rem] sm:max-w-[35rem] h-auto z-50 bg-surface-container-high
|
||||
shadow-2xl shadow-shadow/15 rounded-[1.75rem] p-6 dark:bg-dark-surface-container-high mx-2 ${className}`}
|
||||
initial={{ opacity: 0.5, transform: "scale(1.1)" }}
|
||||
animate={{ opacity: 1, transform: "scale(1)" }}
|
||||
exit={{ opacity: 0 }}
|
||||
transition={{ ease: [0.31, 0.69, 0.3, 1.02], duration: 0.3 }}
|
||||
shadow-2xl shadow-shadow/15 rounded-[1.75rem] p-6 dark:bg-dark-surface-container-high mx-2
|
||||
origin-top ${className} overflow-hidden grid ${isSupported && "grid-rows-[0fr]"}`}
|
||||
initial={{
|
||||
opacity: 0,
|
||||
transform: "translateY(-24px)",
|
||||
gridTemplateRows: isSupported ? undefined : "0fr"
|
||||
}}
|
||||
animate={{
|
||||
opacity: 1,
|
||||
transform: "translateY(0px)",
|
||||
gridTemplateRows: isSupported ? undefined : "1fr"
|
||||
}}
|
||||
exit={{
|
||||
opacity: 0,
|
||||
transform: "translateY(-24px)",
|
||||
gridTemplateRows: isSupported ? undefined : "0fr"
|
||||
}}
|
||||
transition={{ ease: [0.05, 0.7, 0.1, 1.0], duration: 0.35 }}
|
||||
aria-modal="true"
|
||||
ref={dialogRef}
|
||||
>
|
||||
{children}
|
||||
<div className="min-h-0">
|
||||
<motion.div
|
||||
className="origin-top"
|
||||
initial={{ opacity: 0, transform: "translateY(5px)" }}
|
||||
animate={{ opacity: 1, transform: "translateY(0px)" }}
|
||||
exit={{ opacity: 0, transform: "translateY(5px)" }}
|
||||
transition={{
|
||||
ease: [0.05, 0.7, 0.1, 1.0],
|
||||
duration: 0.35
|
||||
}}
|
||||
ref={contentRef}
|
||||
>
|
||||
{children}
|
||||
</motion.div>
|
||||
</div>
|
||||
</motion.div>
|
||||
</div>
|
||||
)}
|
||||
|
@ -58,7 +58,7 @@ export const SearchBox: React.FC<SearchBoxProps> = ({ close = () => {} }) => {
|
||||
type="search"
|
||||
placeholder="搜索"
|
||||
autoComplete="off"
|
||||
autoCapitalize="off"
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
className="bg-transparent h-full w-full focus:outline-none"
|
||||
onKeyDown={handleKeyDown}
|
||||
@ -73,7 +73,7 @@ export const SearchBox: React.FC<SearchBoxProps> = ({ close = () => {} }) => {
|
||||
type="search"
|
||||
placeholder="搜索"
|
||||
autoComplete="off"
|
||||
autoCapitalize="off"
|
||||
autoCapitalize="none"
|
||||
autoCorrect="off"
|
||||
className="bg-transparent h-full w-full focus:outline-none"
|
||||
onKeyDown={handleKeyDown}
|
||||
|
@ -14,7 +14,7 @@ export const ErrorDialog: React.FC<ErrorDialogProps> = ({ children, closeDialog,
|
||||
<>
|
||||
<DialogHeadline>{errorCode ? t(errorCode) : "错误"}</DialogHeadline>
|
||||
<DialogSupportingText>{children}</DialogSupportingText>
|
||||
<DialogButtonGroup>
|
||||
<DialogButtonGroup close={closeDialog}>
|
||||
<DialogButton onClick={closeDialog}>关闭</DialogButton>
|
||||
</DialogButtonGroup>
|
||||
</>
|
||||
|
31
packages/next/components/utils/useKeyboardEvents.tsx
Normal file
31
packages/next/components/utils/useKeyboardEvents.tsx
Normal file
@ -0,0 +1,31 @@
|
||||
import { useEffect, useCallback } from "react";
|
||||
|
||||
export type KeyboardShortcut = {
|
||||
key: string;
|
||||
callback: () => void;
|
||||
preventDefault?: boolean;
|
||||
};
|
||||
|
||||
export function useKeyboardShortcuts(shortcuts: KeyboardShortcut[]): void {
|
||||
const handleKeyDown = useCallback(
|
||||
(event: KeyboardEvent) => {
|
||||
shortcuts.forEach((shortcut) => {
|
||||
if (event.key === shortcut.key) {
|
||||
if (shortcut.preventDefault) {
|
||||
event.preventDefault();
|
||||
}
|
||||
shortcut.callback();
|
||||
}
|
||||
});
|
||||
},
|
||||
[shortcuts]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
document.addEventListener("keydown", handleKeyDown);
|
||||
|
||||
return () => {
|
||||
document.removeEventListener("keydown", handleKeyDown);
|
||||
};
|
||||
}, [handleKeyDown]);
|
||||
}
|
@ -7,6 +7,13 @@
|
||||
"username": "用户名",
|
||||
"password": "密码",
|
||||
"nickname": "昵称",
|
||||
"profile_page": {
|
||||
"role": {
|
||||
"ADMIN": "管理员",
|
||||
"USER": "普通用户",
|
||||
"OWNER": "所有者"
|
||||
}
|
||||
},
|
||||
"backend": {
|
||||
"error": {
|
||||
"incorrect_password": "密码错误。",
|
||||
|
@ -1,15 +1,16 @@
|
||||
import { UserType, sqlCred } from "@cvsa/core";
|
||||
import { UserProfile } from "../userAuth";
|
||||
|
||||
export const getUserBySession = async (sessionID: string) => {
|
||||
const users = await sqlCred<UserType[]>`
|
||||
SELECT u.*
|
||||
FROM users u
|
||||
JOIN login_sessions ls ON u.id = ls.uid
|
||||
WHERE ls.id = ${sessionID};
|
||||
SELECT user_id as id, username, nickname, "role", user_created_at as created_at
|
||||
FROM get_user_by_session_func(${sessionID});
|
||||
`;
|
||||
|
||||
if (users.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const user = users[0];
|
||||
return {
|
||||
uid: user.id,
|
||||
@ -20,4 +21,36 @@ export const getUserBySession = async (sessionID: string) => {
|
||||
};
|
||||
};
|
||||
|
||||
export const getUserSessions = async (sessionID: string) => {};
|
||||
export const queryUserProfile = async (uid: number, sessionID?: string): Promise<UserProfile | null> => {
|
||||
interface Result extends UserType {
|
||||
logged_in: boolean;
|
||||
}
|
||||
const users = await sqlCred<Result[]>`
|
||||
SELECT
|
||||
u.id, u.username, u.nickname, u."role", u.created_at,
|
||||
CASE
|
||||
WHEN (ls.uid IS NOT NULL AND ls.deactivated_at IS NULL AND ls.expire_at > NOW()) THEN TRUE
|
||||
ELSE FALSE
|
||||
END AS logged_in
|
||||
FROM
|
||||
users u
|
||||
LEFT JOIN
|
||||
login_sessions ls ON ls.uid = u.id AND ls.id = ${sessionID || ""}
|
||||
WHERE
|
||||
u.id = ${uid};
|
||||
`;
|
||||
|
||||
if (users.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const user = users[0];
|
||||
return {
|
||||
uid: user.id,
|
||||
username: user.username,
|
||||
nickname: user.nickname,
|
||||
role: user.role,
|
||||
createdAt: user.created_at,
|
||||
isLoggedIn: user.logged_in
|
||||
};
|
||||
};
|
||||
|
@ -1,4 +1,4 @@
|
||||
import axios, { AxiosRequestConfig, AxiosError, Method } from "axios";
|
||||
import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios";
|
||||
|
||||
export class ApiRequestError extends Error {
|
||||
public code: number | undefined;
|
||||
@ -21,10 +21,20 @@ const httpMethods = {
|
||||
patch: axios.patch
|
||||
};
|
||||
|
||||
export function fetcher(url: string): Promise<unknown>;
|
||||
export function fetcher<JSON = unknown>(
|
||||
url: string,
|
||||
init?: Omit<AxiosRequestConfig, "method"> & { method?: Exclude<HttpMethod, "DELETE"> }
|
||||
): Promise<JSON>;
|
||||
export function fetcher(
|
||||
url: string,
|
||||
init: Omit<AxiosRequestConfig, "method"> & { method: "DELETE" }
|
||||
): Promise<AxiosResponse>;
|
||||
|
||||
export async function fetcher<JSON = unknown>(
|
||||
url: string,
|
||||
init?: Omit<AxiosRequestConfig, "method"> & { method?: HttpMethod }
|
||||
): Promise<JSON> {
|
||||
): Promise<JSON | AxiosResponse<any, any>> {
|
||||
const { method = "get", data, ...config } = init || {};
|
||||
|
||||
const fullConfig: AxiosRequestConfig = {
|
||||
@ -38,6 +48,9 @@ export async function fetcher<JSON = unknown>(
|
||||
if (["post", "patch", "put"].includes(m)) {
|
||||
const response = await httpMethods[m](url, data, fullConfig);
|
||||
return response.data;
|
||||
} else if (m === "delete") {
|
||||
const response = await axios.delete(url, fullConfig);
|
||||
return response;
|
||||
} else {
|
||||
const response = await httpMethods[m](url, fullConfig);
|
||||
return response.data;
|
||||
|
@ -1,12 +1,16 @@
|
||||
import { cookies } from "next/headers";
|
||||
import { getUserBySession } from "@/lib/db/user";
|
||||
import { getUserBySession, queryUserProfile } from "@/lib/db/user";
|
||||
|
||||
export interface User {
|
||||
uid: number;
|
||||
username: string;
|
||||
nickname: string | null;
|
||||
role: string;
|
||||
createdAt: string;
|
||||
createdAt: Date;
|
||||
}
|
||||
|
||||
export interface UserProfile extends User {
|
||||
isLoggedIn: boolean;
|
||||
}
|
||||
|
||||
export async function getCurrentUser(): Promise<User | null> {
|
||||
@ -19,6 +23,21 @@ export async function getCurrentUser(): Promise<User | null> {
|
||||
|
||||
return user ?? null;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function getUserProfile(uid: number): Promise<UserProfile | null> {
|
||||
const cookieStore = await cookies();
|
||||
const sessionID = cookieStore.get("session_id");
|
||||
|
||||
try {
|
||||
const user = await queryUserProfile(uid, sessionID?.value);
|
||||
|
||||
return user ?? null;
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "next",
|
||||
"version": "0.1.0",
|
||||
"version": "2.9.1",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"dev": "next dev --turbopack -p 7400",
|
||||
@ -11,16 +11,19 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@cvsa/backend": "^0.5.3",
|
||||
"@cvsa/core": "^0.0.7",
|
||||
"@cvsa/core": "0.0.10",
|
||||
"@mdx-js/loader": "^3.1.0",
|
||||
"@mdx-js/react": "^3.1.0",
|
||||
"@next/mdx": "^15.3.3",
|
||||
"@types/luxon": "^3.6.2",
|
||||
"@types/mdx": "^2.0.13",
|
||||
"axios": "^1.9.0",
|
||||
"date-fns": "^4.1.0",
|
||||
"framer-motion": "^12.15.0",
|
||||
"fumadocs-mdx": "^11.6.6",
|
||||
"i18next": "^25.2.1",
|
||||
"jotai": "^2.12.5",
|
||||
"luxon": "^3.6.1",
|
||||
"next": "^15.3.3",
|
||||
"next-intl": "^4.1.0",
|
||||
"raw-loader": "^4.0.2",
|
||||
@ -28,7 +31,8 @@
|
||||
"react-dom": "^19.1.0",
|
||||
"swr": "^2.3.3",
|
||||
"ua-parser-js": "^2.0.3",
|
||||
"yup": "^1.6.1"
|
||||
"yup": "^1.6.1",
|
||||
"yup-numeric": "^0.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.8.3",
|
||||
|
@ -1,121 +0,0 @@
|
||||
import { Database } from "jsr:@db/sqlite@0.12";
|
||||
import { ensureDir } from "https://deno.land/std@0.113.0/fs/mod.ts";
|
||||
|
||||
// 常量定义
|
||||
const MAX_RETRIES = 3;
|
||||
const API_URL = "https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=50&pn=";
|
||||
const DATABASE_PATH = "./data/main.db";
|
||||
const LOG_DIR = "./logs/bili-info-crawl";
|
||||
const LOG_FILE = `${LOG_DIR}/run-${Date.now() / 1000}.log`;
|
||||
|
||||
// 打开数据库
|
||||
const db = new Database(DATABASE_PATH, { int64: true });
|
||||
|
||||
// 设置日志
|
||||
async function setupLogging() {
|
||||
await ensureDir(LOG_DIR);
|
||||
const logStream = await Deno.open(LOG_FILE, {
|
||||
write: true,
|
||||
create: true,
|
||||
append: true,
|
||||
});
|
||||
|
||||
const redirectConsole =
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(originalConsole: (...args: any[]) => void) =>
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(...args: any[]) => {
|
||||
const message = args.map((
|
||||
arg,
|
||||
) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
||||
originalConsole(message);
|
||||
logStream.write(new TextEncoder().encode(message + "\n"));
|
||||
};
|
||||
|
||||
console.log = redirectConsole(console.log);
|
||||
console.error = redirectConsole(console.error);
|
||||
console.warn = redirectConsole(console.warn);
|
||||
}
|
||||
|
||||
interface Metadata {
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
// 获取最后一次更新的时间
|
||||
function getLastUpdate(): Date {
|
||||
const result = db.prepare(
|
||||
"SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'",
|
||||
).get() as Metadata;
|
||||
return result ? new Date(result.value as string) : new Date(0);
|
||||
}
|
||||
|
||||
// 更新最后更新时间
|
||||
function updateLastUpdate() {
|
||||
const now = new Date().toISOString();
|
||||
db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'")
|
||||
.run(now);
|
||||
}
|
||||
|
||||
// 辅助函数:获取数据
|
||||
// deno-lint-ignore no-explicit-any
|
||||
async function fetchData(pn: number, retries = MAX_RETRIES): Promise<any> {
|
||||
try {
|
||||
const response = await fetch(`${API_URL}${pn}`);
|
||||
if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`);
|
||||
return await response.json();
|
||||
} catch (error) {
|
||||
if (retries > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
return fetchData(pn, retries - 1);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// 插入 aid 到数据库
|
||||
function insertAid(aid: number) {
|
||||
db.prepare(
|
||||
"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
|
||||
).run(aid);
|
||||
}
|
||||
|
||||
// 主函数
|
||||
async function main() {
|
||||
await setupLogging();
|
||||
|
||||
let pn = 1;
|
||||
let shouldContinue = true;
|
||||
const lastUpdate = getLastUpdate();
|
||||
|
||||
while (shouldContinue) {
|
||||
try {
|
||||
const data = await fetchData(pn);
|
||||
const archives = data.data.archives;
|
||||
|
||||
for (const archive of archives) {
|
||||
const pubTime = new Date(archive.pubdate * 1000);
|
||||
if (pubTime > lastUpdate) {
|
||||
insertAid(archive.aid);
|
||||
} else {
|
||||
shouldContinue = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
pn++;
|
||||
console.log(`Fetched page ${pn}`);
|
||||
} catch (error) {
|
||||
console.error(`Error fetching data for pn=${pn}: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// 更新最后更新时间
|
||||
updateLastUpdate();
|
||||
|
||||
// 关闭数据库
|
||||
db.close();
|
||||
}
|
||||
|
||||
// 运行主函数
|
||||
main().catch(console.error);
|
@ -1,223 +0,0 @@
|
||||
import path from "node:path";
|
||||
import { Database } from "jsr:@db/sqlite@0.12";
|
||||
import { getBiliBiliVideoInfo } from "./videoInfo.ts";
|
||||
import { ensureDir } from "https://deno.land/std@0.113.0/fs/mod.ts";
|
||||
|
||||
const aidPath = "./data/2025010104_c30_aids.txt";
|
||||
const db = new Database("./data/main.db", { int64: true });
|
||||
const regions = [
|
||||
"shanghai",
|
||||
"hangzhou",
|
||||
"qingdao",
|
||||
"beijing",
|
||||
"zhangjiakou",
|
||||
"chengdu",
|
||||
"shenzhen",
|
||||
"hohhot",
|
||||
];
|
||||
const logDir = "./logs/bili-info-crawl";
|
||||
const logFile = path.join(logDir, `run-${Date.now() / 1000}.log`);
|
||||
const shouldReadTextFile = false;
|
||||
|
||||
const SECOND = 1000;
|
||||
const SECONDS = SECOND;
|
||||
const MINUTE = 60 * SECONDS;
|
||||
const MINUTES = MINUTE;
|
||||
const IPs = regions.length;
|
||||
|
||||
const rateLimits = [
|
||||
{ window: 5 * MINUTES, maxRequests: 160 * IPs },
|
||||
{ window: 30 * SECONDS, maxRequests: 20 * IPs },
|
||||
{ window: 1.2 * SECOND, maxRequests: 1 * IPs },
|
||||
];
|
||||
|
||||
const requestQueue: number[] = [];
|
||||
|
||||
async function setupLogging() {
|
||||
await ensureDir(logDir);
|
||||
const logStream = await Deno.open(logFile, {
|
||||
write: true,
|
||||
create: true,
|
||||
append: true,
|
||||
});
|
||||
|
||||
const redirectConsole =
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(originalConsole: (...args: any[]) => void) =>
|
||||
// deno-lint-ignore no-explicit-any
|
||||
(...args: any[]) => {
|
||||
const message = args.map((
|
||||
arg,
|
||||
) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
||||
originalConsole(message);
|
||||
logStream.write(new TextEncoder().encode(message + "\n"));
|
||||
};
|
||||
|
||||
console.log = redirectConsole(console.log);
|
||||
console.error = redirectConsole(console.error);
|
||||
console.warn = redirectConsole(console.warn);
|
||||
}
|
||||
|
||||
function isRateLimited(): boolean {
|
||||
const now = Date.now();
|
||||
return rateLimits.some(({ window, maxRequests }) => {
|
||||
const windowStart = now - window;
|
||||
const requestsInWindow = requestQueue.filter((timestamp) => timestamp >= windowStart).length;
|
||||
return requestsInWindow >= maxRequests;
|
||||
});
|
||||
}
|
||||
|
||||
async function readFromText() {
|
||||
const aidRawcontent = await Deno.readTextFile(aidPath);
|
||||
const aids = aidRawcontent
|
||||
.split("\n")
|
||||
.filter((line) => line.length > 0)
|
||||
.map((line) => parseInt(line));
|
||||
|
||||
// if (!db.prepare("SELECT COUNT(*) FROM bili_info_crawl").get()) {
|
||||
// const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')");
|
||||
// aids.forEach((aid) => insertStmt.run(aid));
|
||||
// }
|
||||
|
||||
// 查询数据库中已经存在的 aid
|
||||
const existingAids = db
|
||||
.prepare("SELECT aid FROM bili_info_crawl")
|
||||
.all()
|
||||
.map((row) => row.aid);
|
||||
console.log(existingAids.length);
|
||||
|
||||
// 将 existingAids 转换为 Set 以提高查找效率
|
||||
const existingAidsSet = new Set(existingAids);
|
||||
|
||||
// 找出 aids 数组中不存在于数据库的条目
|
||||
const newAids = aids.filter((aid) => !existingAidsSet.has(aid));
|
||||
|
||||
// 插入这些新条目
|
||||
const insertStmt = db.prepare(
|
||||
"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
|
||||
);
|
||||
newAids.forEach((aid) => insertStmt.run(aid));
|
||||
}
|
||||
|
||||
async function insertAidsToDB() {
|
||||
if (shouldReadTextFile) {
|
||||
await readFromText();
|
||||
}
|
||||
|
||||
const aidsInDB = db
|
||||
.prepare(
|
||||
"SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'",
|
||||
)
|
||||
.all()
|
||||
.map((row) => row.aid) as number[];
|
||||
|
||||
const totalAids = aidsInDB.length;
|
||||
let processedAids = 0;
|
||||
const startTime = Date.now();
|
||||
|
||||
const processAid = async (aid: number) => {
|
||||
try {
|
||||
const res = await getBiliBiliVideoInfo(
|
||||
aid,
|
||||
regions[processedAids % regions.length],
|
||||
);
|
||||
if (res === null) {
|
||||
updateAidStatus(aid, "failed");
|
||||
} else {
|
||||
const rawData = JSON.parse(res);
|
||||
if (rawData.code === 0) {
|
||||
updateAidStatus(
|
||||
aid,
|
||||
"success",
|
||||
rawData.data.View.bvid,
|
||||
JSON.stringify(rawData.data),
|
||||
);
|
||||
} else {
|
||||
updateAidStatus(aid, "error", undefined, res);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Error updating aid ${aid}: ${error}`);
|
||||
updateAidStatus(aid, "failed");
|
||||
} finally {
|
||||
processedAids++;
|
||||
logProgress(aid, processedAids, totalAids, startTime);
|
||||
}
|
||||
};
|
||||
|
||||
const interval = setInterval(async () => {
|
||||
if (aidsInDB.length === 0) {
|
||||
clearInterval(interval);
|
||||
console.log("All aids processed.");
|
||||
return;
|
||||
}
|
||||
if (!isRateLimited()) {
|
||||
const aid = aidsInDB.shift();
|
||||
if (aid !== undefined) {
|
||||
requestQueue.push(Date.now());
|
||||
await processAid(aid);
|
||||
}
|
||||
}
|
||||
}, 50);
|
||||
|
||||
console.log("Starting to process aids...");
|
||||
}
|
||||
|
||||
function updateAidStatus(
|
||||
aid: number,
|
||||
status: string,
|
||||
bvid?: string,
|
||||
data?: string,
|
||||
) {
|
||||
const stmt = db.prepare(`
|
||||
UPDATE bili_info_crawl
|
||||
SET status = ?,
|
||||
${bvid ? "bvid = ?," : ""}
|
||||
${data ? "data = ?," : ""}
|
||||
timestamp = ?
|
||||
WHERE aid = ?
|
||||
`);
|
||||
const params = [
|
||||
status,
|
||||
...(bvid ? [bvid] : []),
|
||||
...(data ? [data] : []),
|
||||
Date.now() / 1000,
|
||||
aid,
|
||||
];
|
||||
stmt.run(...params);
|
||||
}
|
||||
|
||||
function logProgress(
|
||||
aid: number,
|
||||
processedAids: number,
|
||||
totalAids: number,
|
||||
startTime: number,
|
||||
) {
|
||||
const elapsedTime = Date.now() - startTime;
|
||||
const elapsedSeconds = Math.floor(elapsedTime / 1000);
|
||||
const elapsedMinutes = Math.floor(elapsedSeconds / 60);
|
||||
const elapsedHours = Math.floor(elapsedMinutes / 60);
|
||||
|
||||
const remainingAids = totalAids - processedAids;
|
||||
const averageTimePerAid = elapsedTime / processedAids;
|
||||
const eta = remainingAids * averageTimePerAid;
|
||||
const etaSeconds = Math.floor(eta / 1000);
|
||||
const etaMinutes = Math.floor(etaSeconds / 60);
|
||||
const etaHours = Math.floor(etaMinutes / 60);
|
||||
|
||||
const progress = `${processedAids}/${totalAids}, ${
|
||||
((processedAids / totalAids) * 100).toFixed(
|
||||
2,
|
||||
)
|
||||
}%, elapsed ${elapsedHours.toString().padStart(2, "0")}:${(elapsedMinutes % 60).toString().padStart(2, "0")}:${
|
||||
(
|
||||
elapsedSeconds % 60
|
||||
)
|
||||
.toString()
|
||||
.padStart(2, "0")
|
||||
}, ETA ${etaHours}h${(etaMinutes % 60).toString().padStart(2, "0")}m`;
|
||||
console.log(`Updated aid ${aid}, ${progress}`);
|
||||
}
|
||||
|
||||
await setupLogging();
|
||||
insertAidsToDB();
|
@ -1,60 +0,0 @@
|
||||
export async function getBiliBiliVideoInfo(
|
||||
bvidORaid?: string | number,
|
||||
region: string = "hangzhou",
|
||||
) {
|
||||
const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined;
|
||||
const aid = typeof bvidORaid === "number" ? bvidORaid : undefined;
|
||||
|
||||
const baseURL = "https://api.bilibili.com/x/web-interface/view/detail";
|
||||
const urlObject = new URL(baseURL);
|
||||
|
||||
if (aid) {
|
||||
urlObject.searchParams.append("aid", aid.toString());
|
||||
const finalURL = urlObject.toString();
|
||||
return await proxyRequestWithRegion(finalURL, region);
|
||||
} else if (bvid) {
|
||||
urlObject.searchParams.append("bvid", bvid);
|
||||
const finalURL = urlObject.toString();
|
||||
return await proxyRequestWithRegion(finalURL, region);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function proxyRequestWithRegion(
|
||||
url: string,
|
||||
region: string,
|
||||
): Promise<any | null> {
|
||||
const td = new TextDecoder();
|
||||
// aliyun configure set --access-key-id $ALIYUN_AK --access-key-secret $ALIYUN_SK --region cn-shenzhen --profile CVSA-shenzhen --mode AK
|
||||
const p = await new Deno.Command("aliyun", {
|
||||
args: [
|
||||
"fc",
|
||||
"POST",
|
||||
`/2023-03-30/functions/proxy-${region}/invocations`,
|
||||
"--qualifier",
|
||||
"LATEST",
|
||||
"--header",
|
||||
"Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;",
|
||||
"--body",
|
||||
JSON.stringify({ url: url }),
|
||||
"--profile",
|
||||
`CVSA-${region}`,
|
||||
],
|
||||
}).output();
|
||||
try {
|
||||
const out = td.decode(p.stdout);
|
||||
const rawData = JSON.parse(out);
|
||||
if (rawData.statusCode !== 200) {
|
||||
console.error(
|
||||
`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`,
|
||||
);
|
||||
return null;
|
||||
} else {
|
||||
return JSON.parse(rawData.body);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error proxying request ${url} to ${region}: ${e}`);
|
||||
return null;
|
||||
}
|
||||
}
|
32
src/metadataArchive.ts
Normal file
32
src/metadataArchive.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import arg from "arg";
|
||||
//import { getVideoDetails } from "@crawler/net/getVideoDetails";
|
||||
import logger from "@core/log/logger";
|
||||
|
||||
const quit = (reason: string) => {
|
||||
logger.error(reason);
|
||||
process.exit();
|
||||
};
|
||||
|
||||
const args = arg({
|
||||
"--aids": String // --port <number> or --port=<number>
|
||||
});
|
||||
|
||||
const aidsFileName = args["--aids"];
|
||||
|
||||
if (!aidsFileName) {
|
||||
quit("Missing --aids <file_path>");
|
||||
}
|
||||
|
||||
const aidsFile = Bun.file(aidsFileName!);
|
||||
const fileExists = await aidsFile.exists();
|
||||
if (!fileExists) {
|
||||
quit(`${aidsFile} does not exist.`);
|
||||
}
|
||||
|
||||
const aidsText = await aidsFile.text();
|
||||
const aids = aidsText
|
||||
.split("\n")
|
||||
.map((line) => parseInt(line))
|
||||
.filter((num) => !Number.isNaN(num));
|
||||
|
||||
logger.log(`Read ${aids.length} aids.`);
|
35
tsconfig.json
Normal file
35
tsconfig.json
Normal file
@ -0,0 +1,35 @@
|
||||
{
|
||||
"include": ["**/*.ts"],
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@core/*": ["./packages/core/*"],
|
||||
"@crawler/*": ["./packages/crawler/*"]
|
||||
},
|
||||
// Environment setup & latest features
|
||||
"lib": ["ESNext"],
|
||||
"target": "ESNext",
|
||||
"module": "Preserve",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
// Best practices
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"noImplicitOverride": true,
|
||||
|
||||
// Some stricter flags (disabled by default)
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue
Block a user