diff --git a/bun.lock b/bun.lock index 8a510ef..a84c02a 100644 --- a/bun.lock +++ b/bun.lock @@ -41,10 +41,12 @@ "version": "0.0.10", "dependencies": { "@koshnic/ratelimit": "^1.0.3", + "@types/luxon": "^3.7.1", "chalk": "^5.4.1", "drizzle-orm": "^0.44.4", "ioredis": "^5.6.1", "logform": "^2.7.0", + "luxon": "^3.7.2", "postgres": "^3.4.5", "winston": "^3.17.0", }, diff --git a/packages/backend/db/latestSnapshots.ts b/packages/backend/db/latestSnapshots.ts index 1d9d79b..4535bec 100644 --- a/packages/backend/db/latestSnapshots.ts +++ b/packages/backend/db/latestSnapshots.ts @@ -1,5 +1,5 @@ import { sql } from "@core/db/dbNew"; -import type { LatestSnapshotType } from "@core/db/schema.d.ts"; +import type { LatestSnapshotType } from "@core/db/schema"; export async function getVideosInViewsRange(minViews: number, maxViews: number) { return sql` diff --git a/packages/backend/db/snapshots.ts b/packages/backend/db/snapshots.ts index 83ea7b5..580e823 100644 --- a/packages/backend/db/snapshots.ts +++ b/packages/backend/db/snapshots.ts @@ -1,5 +1,5 @@ import { sql } from "@core/db/dbNew"; -import type { VideoSnapshotType } from "@core/db/schema.d.ts"; +import type { VideoSnapshotType } from "@core/db/schema"; export async function getVideoSnapshots( aid: number, diff --git a/packages/backend/lib/auth/captchaDifficulty.ts b/packages/backend/lib/auth/captchaDifficulty.ts index f217688..162dc89 100644 --- a/packages/backend/lib/auth/captchaDifficulty.ts +++ b/packages/backend/lib/auth/captchaDifficulty.ts @@ -1,7 +1,7 @@ import { Psql } from "@core/db/psql"; -import { SlidingWindow } from "@core/mq/slidingWindow.ts"; -import { redis } from "@core/db/redis.ts"; -import { getIdentifier } from "@/middleware/rateLimiters.ts"; +import { SlidingWindow } from "@core/mq/slidingWindow"; +import { redis } from "@core/db/redis"; +import { getIdentifier } from "@/middleware/rateLimiters"; import { Context } from "hono"; type seconds = number; diff --git a/packages/backend/middleware/bodyLimits.ts b/packages/backend/middleware/bodyLimits.ts index 4b3b34e..389c09b 100644 --- a/packages/backend/middleware/bodyLimits.ts +++ b/packages/backend/middleware/bodyLimits.ts @@ -1,10 +1,10 @@ import { bodyLimit } from "hono/body-limit"; -import { ErrorResponse } from "../src/schema"; +import { ErrorResponse } from "@/src/schema"; export const bodyLimitForPing = bodyLimit({ maxSize: 14000, onError: (c) => { - const res: ErrorResponse = { + const res: ErrorResponse = { message: "Body too large", errors: ["Body should not be larger than 14kB."], code: "BODY_TOO_LARGE" diff --git a/packages/backend/middleware/captcha.ts b/packages/backend/middleware/captcha.ts index 60b0d40..dde000a 100644 --- a/packages/backend/middleware/captcha.ts +++ b/packages/backend/middleware/captcha.ts @@ -1,15 +1,15 @@ import { Context, Next } from "hono"; import { ErrorResponse } from "src/schema"; -import { SlidingWindow } from "@core/mq/slidingWindow.ts"; -import { getCaptchaConfigMaxDuration, getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; -import { sqlCred } from "@core/db/dbNew.ts"; -import { redis } from "@core/db/redis.ts"; +import { SlidingWindow } from "@core/mq/slidingWindow"; +import { getCaptchaConfigMaxDuration, getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty"; +import { sqlCred } from "@core/db/dbNew"; +import { redis } from "@core/db/redis"; import { verify } from "hono/jwt"; import { JwtTokenInvalid, JwtTokenExpired } from "hono/utils/jwt/types"; -import { getJWTsecret } from "@/lib/auth/getJWTsecret.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; +import { getJWTsecret } from "@/lib/auth/getJWTsecret"; +import { lockManager } from "@core/mq/lockManager"; import { object, string, number, ValidationError } from "yup"; -import { getIdentifier } from "@/middleware/rateLimiters.ts"; +import { getIdentifier } from "@/middleware/rateLimiters"; const tokenSchema = object({ exp: number().integer(), diff --git a/packages/backend/middleware/rateLimiters.ts b/packages/backend/middleware/rateLimiters.ts index 00531f7..b57a801 100644 --- a/packages/backend/middleware/rateLimiters.ts +++ b/packages/backend/middleware/rateLimiters.ts @@ -1,10 +1,10 @@ import type { BlankEnv } from "hono/types"; import { getConnInfo } from "hono/bun"; import { Context, Next } from "hono"; -import { generateRandomId } from "@core/lib/randomID.ts"; +import { generateRandomId } from "@core/lib/randomID"; import { RateLimiter } from "@koshnic/ratelimit"; import { ErrorResponse } from "@/src/schema"; -import { redis } from "@core/db/redis.ts"; +import { redis } from "@core/db/redis"; export const getUserIP = (c: Context) => { let ipAddr = null; diff --git a/packages/backend/routes/captcha/[id]/result/GET.ts b/packages/backend/routes/captcha/[id]/result/GET.ts index 0c9c2e4..caf9324 100644 --- a/packages/backend/routes/captcha/[id]/result/GET.ts +++ b/packages/backend/routes/captcha/[id]/result/GET.ts @@ -1,10 +1,10 @@ import { Context } from "hono"; import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import { sign } from "hono/jwt"; -import { generateRandomId } from "@core/lib/randomID.ts"; -import { getJWTsecret } from "lib/auth/getJWTsecret.ts"; +import { generateRandomId } from "@core/lib/randomID"; +import { getJWTsecret } from "lib/auth/getJWTsecret"; interface CaptchaResponse { success: boolean; diff --git a/packages/backend/routes/captcha/difficulty/GET.ts b/packages/backend/routes/captcha/difficulty/GET.ts index ce1d339..1f2d420 100644 --- a/packages/backend/routes/captcha/difficulty/GET.ts +++ b/packages/backend/routes/captcha/difficulty/GET.ts @@ -1,8 +1,8 @@ -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import { object, string, ValidationError } from "yup"; import { ErrorResponse } from "src/schema"; -import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; -import { sqlCred } from "@core/db/dbNew.ts"; +import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty"; +import { sqlCred } from "@core/db/dbNew"; const queryParamsSchema = object({ route: string().matches(/(?:GET|POST|PUT|PATCH|DELETE)-\/.*/g) diff --git a/packages/backend/routes/captcha/session/POST.ts b/packages/backend/routes/captcha/session/POST.ts index 0836368..cf6dd43 100644 --- a/packages/backend/routes/captcha/session/POST.ts +++ b/packages/backend/routes/captcha/session/POST.ts @@ -1,6 +1,6 @@ -import { createHandlers } from "src/utils.ts"; -import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty.ts"; -import { sqlCred } from "@core/db/dbNew.ts"; +import { createHandlers } from "src/utils"; +import { getCurrentCaptchaDifficulty } from "@/lib/auth/captchaDifficulty"; +import { sqlCred } from "@core/db/dbNew"; import { object, string, ValidationError } from "yup"; import { CaptchaSessionResponse, ErrorResponse } from "@/src/schema"; import type { ContentfulStatusCode } from "hono/utils/http-status"; diff --git a/packages/backend/routes/index.ts b/packages/backend/routes/index.ts index de83d72..6d62b38 100644 --- a/packages/backend/routes/index.ts +++ b/packages/backend/routes/index.ts @@ -1,6 +1,6 @@ -import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "lib/const/singers.ts"; -import { VERSION } from "src/main.ts"; -import { createHandlers } from "src/utils.ts"; +import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "lib/const/singers"; +import { VERSION } from "src/main"; +import { createHandlers } from "src/utils"; export const rootHandler = createHandlers((c) => { let singer: Singer | Singer[]; diff --git a/packages/backend/routes/login/session/POST.ts b/packages/backend/routes/login/session/POST.ts index ee4b1be..7872baa 100644 --- a/packages/backend/routes/login/session/POST.ts +++ b/packages/backend/routes/login/session/POST.ts @@ -1,7 +1,7 @@ import { Context } from "hono"; import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse, LoginResponse } from "src/schema"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import { sqlCred } from "@core/db/dbNew"; import { object, string, ValidationError } from "yup"; import { setCookie } from "hono/cookie"; @@ -27,12 +27,12 @@ export const loginHandler = createHandlers( `; if (result.length === 0) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: `User does not exist.`, errors: [`User ${username} does not exist.`], code: "ENTITY_NOT_FOUND" }; - return c.json>(response, 400); + return c.json(response, 400); } const storedPassword = result[0].password; @@ -43,7 +43,7 @@ export const loginHandler = createHandlers( const passwordAreSame = await Argon2id.verify(storedPassword, submittedPassword); if (!passwordAreSame) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Incorrect password.", errors: [], i18n: { @@ -51,7 +51,7 @@ export const loginHandler = createHandlers( }, code: "INVALID_CREDENTIALS" }; - return c.json>(response, 401); + return c.json(response, 401); } const sessionID = await createLoginSession(uid, c); @@ -79,26 +79,26 @@ export const loginHandler = createHandlers( return c.json(response, 200); } catch (e) { if (e instanceof ValidationError) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Invalid registration data.", errors: e.errors, code: "INVALID_PAYLOAD" }; - return c.json>(response, 400); + return c.json(response, 400); } else if (e instanceof SyntaxError) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [e.message], code: "INVALID_FORMAT" }; - return c.json>(response, 400); + return c.json(response, 400); } else { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Unknown error.", errors: [(e as Error).message], code: "UNKNOWN_ERROR" }; - return c.json>(response, 500); + return c.json(response, 500); } } } diff --git a/packages/backend/routes/ping/index.ts b/packages/backend/routes/ping/index.ts index c1caecb..bb151e5 100644 --- a/packages/backend/routes/ping/index.ts +++ b/packages/backend/routes/ping/index.ts @@ -1,6 +1,6 @@ -import { getClientIP } from "middleware/logger.ts"; -import { createHandlers } from "src/utils.ts"; -import { VERSION } from "src/main.ts"; +import { getClientIP } from "middleware/logger"; +import { createHandlers } from "src/utils"; +import { VERSION } from "src/main"; export const pingHandler = createHandlers(async (c) => { const requestHeaders = c.req.raw.headers; diff --git a/packages/backend/routes/session/[id]/DELETE.ts b/packages/backend/routes/session/[id]/DELETE.ts index 994034e..0bc8b64 100644 --- a/packages/backend/routes/session/[id]/DELETE.ts +++ b/packages/backend/routes/session/[id]/DELETE.ts @@ -1,9 +1,9 @@ import { Context } from "hono"; import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import { sqlCred } from "@core/db/dbNew"; -import { object, string, ValidationError } from "yup"; +import { ValidationError } from "yup"; import { setCookie } from "hono/cookie"; const loginSessionExists = async (sessionID: string) => { @@ -22,12 +22,12 @@ export const logoutHandler = createHandlers(async (c: Context = { + const response: ErrorResponse = { message: "Cannot found given session_id.", errors: [`Session ${session_id} not found`], code: "ENTITY_NOT_FOUND" }; - return c.json>(response, 404); + return c.json(response, 404); } await sqlCred` @@ -50,26 +50,26 @@ export const logoutHandler = createHandlers(async (c: Context = { + const response: ErrorResponse = { message: "Invalid registration data.", errors: e.errors, code: "INVALID_PAYLOAD" }; - return c.json>(response, 400); + return c.json(response, 400); } else if (e instanceof SyntaxError) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [e.message], code: "INVALID_FORMAT" }; - return c.json>(response, 400); + return c.json(response, 400); } else { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Unknown error.", errors: [(e as Error).message], code: "UNKNOWN_ERROR" }; - return c.json>(response, 500); + return c.json(response, 500); } } }); diff --git a/packages/backend/routes/user/POST.ts b/packages/backend/routes/user/POST.ts index f30489c..ba2d20f 100644 --- a/packages/backend/routes/user/POST.ts +++ b/packages/backend/routes/user/POST.ts @@ -1,9 +1,9 @@ -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import Argon2id from "@rabbit-company/argon2id"; import { object, string, ValidationError } from "yup"; import type { Context } from "hono"; import type { Bindings, BlankEnv, BlankInput } from "hono/types"; -import { sqlCred } from "@core/db/dbNew.ts"; +import { sqlCred } from "@core/db/dbNew"; import { ErrorResponse, SignUpResponse } from "src/schema"; import { generateRandomId } from "@core/lib/randomID"; import { getUserIP } from "@/middleware/rateLimiters"; @@ -79,7 +79,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => { const uid = await getUserIDByName(username); if (!uid) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Cannot find registered user.", errors: [`Cannot find user ${username} in table 'users'.`], code: "ENTITY_NOT_FOUND", @@ -90,7 +90,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => { } } }; - return c.json>(response, 500); + return c.json(response, 500); } const sessionID = await createLoginSession(uid, c); @@ -115,26 +115,26 @@ export const registerHandler = createHandlers(async (c: ContextType) => { return c.json(response, 201); } catch (e) { if (e instanceof ValidationError) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Invalid registration data.", errors: e.errors, code: "INVALID_PAYLOAD" }; - return c.json>(response, 400); + return c.json(response, 400); } else if (e instanceof SyntaxError) { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Invalid JSON payload.", errors: [e.message], code: "INVALID_FORMAT" }; - return c.json>(response, 400); + return c.json(response, 400); } else { - const response: ErrorResponse = { + const response: ErrorResponse = { message: "Unknown error.", errors: [(e as Error).message], code: "UNKNOWN_ERROR" }; - return c.json>(response, 500); + return c.json(response, 500); } } }); diff --git a/packages/backend/routes/user/session/[id]/GET.ts b/packages/backend/routes/user/session/[id]/GET.ts index 78b9c3d..de1b6f5 100644 --- a/packages/backend/routes/user/session/[id]/GET.ts +++ b/packages/backend/routes/user/session/[id]/GET.ts @@ -1,7 +1,7 @@ import { Context } from "hono"; import { Bindings, BlankEnv } from "hono/types"; import { ErrorResponse } from "src/schema"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import { sqlCred } from "@core/db/dbNew"; import { UserType } from "@core/db/schema"; diff --git a/packages/backend/routes/video/[id]/info.ts b/packages/backend/routes/video/[id]/info.ts index 3e05493..af3bd53 100644 --- a/packages/backend/routes/video/[id]/info.ts +++ b/packages/backend/routes/video/[id]/info.ts @@ -1,14 +1,14 @@ -import logger from "@core/log/logger.ts"; -import { redis } from "@core/db/redis.ts"; -import { sql } from "@core/db/dbNew.ts"; +import logger from "@core/log"; +import { redis } from "@core/db/redis"; +import { sql } from "@core/db/dbNew"; import { number, ValidationError } from "yup"; -import { createHandlers } from "@/src/utils.ts"; -import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo.ts"; -import { idSchema } from "./snapshots.ts"; -import { NetSchedulerError } from "@core/net/delegate.ts"; +import { createHandlers } from "@/src/utils"; +import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo"; +import { idSchema } from "./snapshots"; +import { NetSchedulerError } from "@core/net/delegate"; import type { Context } from "hono"; import type { BlankEnv, BlankInput } from "hono/types"; -import type { VideoInfoData } from "@core/net/bilibili.d.ts"; +import type { VideoInfoData } from "@core/net/bilibili"; import { startTime, endTime } from "hono/timing"; const CACHE_EXPIRATION_SECONDS = 60; diff --git a/packages/backend/routes/video/[id]/snapshots.ts b/packages/backend/routes/video/[id]/snapshots.ts index e738e50..3abb5b3 100644 --- a/packages/backend/routes/video/[id]/snapshots.ts +++ b/packages/backend/routes/video/[id]/snapshots.ts @@ -1,8 +1,8 @@ import type { Context } from "hono"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import type { BlankEnv, BlankInput } from "hono/types"; -import { getVideoSnapshots, getVideoSnapshotsByBV } from "db/snapshots.ts"; -import type { VideoSnapshotType } from "@core/db/schema.d.ts"; +import { getVideoSnapshots, getVideoSnapshotsByBV } from "db/snapshots"; +import type { VideoSnapshotType } from "@core/db/schema"; import { boolean, mixed, number, object, ValidationError } from "yup"; import { ErrorResponse } from "src/schema"; import { startTime, endTime } from "hono/timing"; diff --git a/packages/backend/routes/videos/GET.ts b/packages/backend/routes/videos/GET.ts index 65ba5e8..ea2eb32 100644 --- a/packages/backend/routes/videos/GET.ts +++ b/packages/backend/routes/videos/GET.ts @@ -1,5 +1,5 @@ import type { Context } from "hono"; -import { createHandlers } from "src/utils.ts"; +import { createHandlers } from "src/utils"; import type { BlankEnv, BlankInput } from "hono/types"; import { number, object, ValidationError } from "yup"; import { ErrorResponse } from "src/schema"; diff --git a/packages/backend/src/main.ts b/packages/backend/src/main.ts index 4c9973a..607a1e1 100644 --- a/packages/backend/src/main.ts +++ b/packages/backend/src/main.ts @@ -1,9 +1,9 @@ import { Hono } from "hono"; import type { TimingVariables } from "hono/timing"; -import { startServer } from "./startServer.ts"; -import { configureRoutes } from "./routing.ts"; -import { configureMiddleWares } from "./middleware.ts"; -import { notFoundRoute } from "routes/404.ts"; +import { startServer } from "./startServer"; +import { configureRoutes } from "./routing"; +import { configureMiddleWares } from "./middleware"; +import { notFoundRoute } from "routes/404"; type Variables = TimingVariables; const app = new Hono<{ Variables: Variables }>(); diff --git a/packages/backend/src/middleware.ts b/packages/backend/src/middleware.ts index 956f37f..8733a83 100644 --- a/packages/backend/src/middleware.ts +++ b/packages/backend/src/middleware.ts @@ -2,13 +2,13 @@ import { Hono } from "hono"; import { timing } from "hono/timing"; import { Variables } from "hono/types"; import { pingHandler } from "routes/ping"; -import { logger } from "middleware/logger.ts"; +import { logger } from "middleware/logger"; import { corsMiddleware } from "@/middleware/cors"; -import { contentType } from "middleware/contentType.ts"; -import { captchaMiddleware } from "middleware/captcha.ts"; -import { bodyLimitForPing } from "middleware/bodyLimits.ts"; -import { registerRateLimiter } from "middleware/rateLimiters.ts"; -import { preetifyResponse } from "middleware/preetifyResponse.ts"; +import { contentType } from "middleware/contentType"; +import { captchaMiddleware } from "middleware/captcha"; +import { bodyLimitForPing } from "middleware/bodyLimits"; +import { registerRateLimiter } from "middleware/rateLimiters"; +import { preetifyResponse } from "middleware/preetifyResponse"; export function configureMiddleWares(app: Hono<{ Variables: Variables }>) { app.use("*", corsMiddleware); diff --git a/packages/backend/src/routing.ts b/packages/backend/src/routing.ts index 9f44185..2b36ea6 100644 --- a/packages/backend/src/routing.ts +++ b/packages/backend/src/routing.ts @@ -5,7 +5,7 @@ import { videoInfoHandler, getSnapshotsHanlder } from "routes/video"; import { Hono } from "hono"; import { Variables } from "hono/types"; import { createCaptchaSessionHandler, verifyChallengeHandler } from "routes/captcha"; -import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET.ts"; +import { getCaptchaDifficultyHandler } from "routes/captcha/difficulty/GET"; import { getVideosHanlder } from "@/routes/videos"; import { loginHandler } from "@/routes/login/session/POST"; import { logoutHandler } from "@/routes/session"; diff --git a/packages/backend/src/schema.d.ts b/packages/backend/src/schema.d.ts index 7d01dd6..b94c064 100644 --- a/packages/backend/src/schema.d.ts +++ b/packages/backend/src/schema.d.ts @@ -15,7 +15,7 @@ export type ErrorCode = export interface ErrorResponse { code: ErrorCode; message: string; - errors: E[] = []; + errors: E[]; i18n?: { key: string; values?: { diff --git a/packages/core/db/index.ts b/packages/core/db/index.ts new file mode 100644 index 0000000..b38228a --- /dev/null +++ b/packages/core/db/index.ts @@ -0,0 +1 @@ +export * from "./snapshots" \ No newline at end of file diff --git a/packages/core/db/snapshots/getClosetSnapshot.ts b/packages/core/db/snapshots/getClosetSnapshot.ts new file mode 100644 index 0000000..ba9016b --- /dev/null +++ b/packages/core/db/snapshots/getClosetSnapshot.ts @@ -0,0 +1,30 @@ +import { dbMain } from "@core/drizzle"; +import { sql } from "drizzle-orm"; + +export const getClosestSnapshot = async (aid: number, targetTime: Date) => { + const closest = await dbMain.execute<{ created_at: Date; views: number }>(sql` + SELECT created_at, views + FROM ( + (SELECT created_at, views, 'later' AS type + FROM video_snapshot + WHERE aid = ${aid} + AND created_at >= ${targetTime.toISOString()} + ORDER BY created_at + LIMIT 1) + UNION ALL + (SELECT created_at, views, 'earlier' AS type + FROM video_snapshot + WHERE aid = ${aid} + AND created_at <= ${targetTime.toISOString()} + ORDER BY created_at DESC + LIMIT 1) + ) AS combined + ORDER BY + CASE + WHEN created_at >= ${targetTime.toISOString()} THEN created_at -${targetTime.toISOString()} + ELSE ${targetTime.toISOString()} - created_at + END + LIMIT 1; + `); + return closest[0] || null; +}; diff --git a/packages/core/db/snapshots/getLatestSnapshot.ts b/packages/core/db/snapshots/getLatestSnapshot.ts new file mode 100644 index 0000000..a4e4253 --- /dev/null +++ b/packages/core/db/snapshots/getLatestSnapshot.ts @@ -0,0 +1,11 @@ +import { dbMain } from "@core/drizzle"; +import { latestVideoSnapshot } from "@core/drizzle/main/schema"; +import { eq } from "drizzle-orm"; + +export const getLatestSnapshot = async (aid: number) =>{ + const result = await dbMain.select().from(latestVideoSnapshot).where(eq(latestVideoSnapshot.aid, aid)).limit(1); + if (result.length === 0) { + return null; + } + return result[0]; +} \ No newline at end of file diff --git a/packages/core/db/snapshots/index.ts b/packages/core/db/snapshots/index.ts new file mode 100644 index 0000000..60906da --- /dev/null +++ b/packages/core/db/snapshots/index.ts @@ -0,0 +1,2 @@ +export * from "./getLatestSnapshot"; +export * from "./getClosetSnapshot"; \ No newline at end of file diff --git a/packages/core/db/snapshots/milestone.ts b/packages/core/db/snapshots/milestone.ts new file mode 100644 index 0000000..5178860 --- /dev/null +++ b/packages/core/db/snapshots/milestone.ts @@ -0,0 +1,30 @@ +import { MINUTE, HOUR, getClosetMilestone, getMileStoneETAfactor, truncate } from "@core/lib"; +import { getLatestSnapshot, getClosestSnapshot } from "@core/db"; + +export const getShortTermETA = async (aid: number) => { + const DELTA = 1e-5; + let minETAHours = Infinity; + const timeIntervals = [20 * MINUTE, HOUR, 3 * HOUR, 6 * HOUR, 24 * HOUR, 72 * HOUR, 168 * HOUR]; + const currentTimestamp = new Date().getTime(); + const latestSnapshot = await getLatestSnapshot(aid); + for (const timeInterval of timeIntervals) { + const date = new Date(currentTimestamp - timeInterval); + const snapshot = await getClosestSnapshot(aid, date); + if (!snapshot) continue; + const latestSnapshotTime = new Date(latestSnapshot.time).getTime(); + const currentSnapshotTime = new Date(snapshot.created_at).getTime(); + const hoursDiff = (latestSnapshotTime - currentSnapshotTime) / HOUR; + const viewsDiff = latestSnapshot.views - snapshot.views; + if (viewsDiff <= 0) continue; + const speed = viewsDiff / (hoursDiff + DELTA); + const target = getClosetMilestone(latestSnapshot.views); + const viewsToIncrease = target - latestSnapshot.views; + const eta = viewsToIncrease / (speed + DELTA); + let factor = getMileStoneETAfactor(viewsToIncrease); + factor = truncate(factor, 4.5, 100); + const adjustedETA = eta / factor; + if (adjustedETA < minETAHours) { + minETAHours = adjustedETA; + } + } +}; diff --git a/packages/core/lib/index.ts b/packages/core/lib/index.ts new file mode 100644 index 0000000..76ccac8 --- /dev/null +++ b/packages/core/lib/index.ts @@ -0,0 +1,4 @@ +export * from "./math"; +export * from "./milestone"; +export * from "./randomID"; +export * from "./time"; diff --git a/packages/core/lib/math.ts b/packages/core/lib/math.ts new file mode 100644 index 0000000..f81bc2d --- /dev/null +++ b/packages/core/lib/math.ts @@ -0,0 +1,3 @@ +export const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); + +export const truncate = (num: number, min: number, max: number) => Math.max(min, Math.min(num, max)); diff --git a/packages/core/lib/milestone.ts b/packages/core/lib/milestone.ts new file mode 100644 index 0000000..e607e73 --- /dev/null +++ b/packages/core/lib/milestone.ts @@ -0,0 +1,20 @@ +import { log } from "@core/lib"; + +export const getMileStoneETAfactor = (x: number) => { + const a = 1.054; + const b = 4.5; + const c = 100; + const u = 0.601; + const g = 455; + if (x > g) { + return log(b / log(x + 1), a); + } else { + return log(b / log(x + c), a) + u; + } +}; + +export const getClosetMilestone = (views: number) => { + if (views < 100000) return 100000; + if (views < 1000000) return 1000000; + return Math.ceil(views / 1000000) * 1000000; +}; diff --git a/packages/core/const/time.ts b/packages/core/lib/time.ts similarity index 100% rename from packages/core/const/time.ts rename to packages/core/lib/time.ts diff --git a/packages/core/log/logger.ts b/packages/core/log/index.ts similarity index 100% rename from packages/core/log/logger.ts rename to packages/core/log/index.ts diff --git a/packages/core/log/test.ts b/packages/core/log/test.ts deleted file mode 100644 index 0c48452..0000000 --- a/packages/core/log/test.ts +++ /dev/null @@ -1,12 +0,0 @@ -import logger from "@core/log/logger.ts"; - -logger.error(Error("test error"), "test service"); -logger.debug(`some string`); -logger.debug(`hello`, "servicename"); -logger.debug(`hello`, "servicename", "codepath.ts"); -logger.log("something"); -logger.log("foo", "service"); -logger.log("foo", "db", "insert.ts"); -logger.warn("warn"); -logger.error("error"); -logger.verbose("error"); diff --git a/packages/core/mq/lockManager.ts b/packages/core/mq/lockManager.ts index 129001a..a11e22f 100644 --- a/packages/core/mq/lockManager.ts +++ b/packages/core/mq/lockManager.ts @@ -1,5 +1,5 @@ import { Redis } from "ioredis"; -import { redis } from "@core/db/redis.ts"; +import { redis } from "@core/db/redis"; class LockManager { private redis: Redis; diff --git a/packages/core/mq/multipleRateLimiter.ts b/packages/core/mq/multipleRateLimiter.ts index d94c084..6be42d6 100644 --- a/packages/core/mq/multipleRateLimiter.ts +++ b/packages/core/mq/multipleRateLimiter.ts @@ -1,5 +1,5 @@ import { RateLimiter as Limiter } from "@koshnic/ratelimit"; -import { redis } from "@core/db/redis.ts"; +import { redis } from "@core/db/redis"; export interface RateLimiterConfig { duration: number; diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index a1fe8d6..1b784b4 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,7 +1,7 @@ -import logger from "@core/log/logger.ts"; -import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter.ts"; +import logger from "@core/log"; +import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter"; import { ReplyError } from "ioredis"; -import { SECOND } from "@core/const/time.ts"; +import { SECOND } from "@core/lib"; import { spawn, SpawnOptions } from "child_process"; export function spawnPromise( diff --git a/packages/core/net/getVideoInfo.ts b/packages/core/net/getVideoInfo.ts index e13f599..b842231 100644 --- a/packages/core/net/getVideoInfo.ts +++ b/packages/core/net/getVideoInfo.ts @@ -1,6 +1,6 @@ -import networkDelegate from "@core/net/delegate.ts"; -import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d.ts"; -import logger from "@core/log/logger.ts"; +import networkDelegate from "@core/net/delegate"; +import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d"; +import logger from "@core/log"; /* * Fetch video metadata from bilibili API diff --git a/packages/core/package.json b/packages/core/package.json index 5e26518..4bd3d98 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -8,11 +8,13 @@ }, "dependencies": { "@koshnic/ratelimit": "^1.0.3", + "@types/luxon": "^3.7.1", "chalk": "^5.4.1", + "drizzle-orm": "^0.44.4", "ioredis": "^5.6.1", "logform": "^2.7.0", + "luxon": "^3.7.2", "postgres": "^3.4.5", - "drizzle-orm": "^0.44.4", "winston": "^3.17.0" }, "devDependencies": { diff --git a/packages/core/test/lib/randomID.test.ts b/packages/core/test/lib/randomID.test.ts deleted file mode 100644 index 8f69dcd..0000000 --- a/packages/core/test/lib/randomID.test.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { generateRandomId } from "@core/lib/randomID.ts"; - -describe("generateRandomId", () => { - it("should generate an ID of the specified length", () => { - const length = 15; - const id = generateRandomId(length); - expect(id).toHaveLength(length); - }); - - it("should generate an ID containing only allowed characters", () => { - const allowedChars = "abcdefghijkmnpqrstuvwxyzABCDEFGHJKLMNPQRSTUVWXYZ23456789"; - const id = generateRandomId(20); - for (const char of id) { - expect(allowedChars).toContain(char); - } - }); -}); diff --git a/packages/crawler/db/bilibili_metadata.ts b/packages/crawler/db/bilibili_metadata.ts index b6b363f..c009bdd 100644 --- a/packages/crawler/db/bilibili_metadata.ts +++ b/packages/crawler/db/bilibili_metadata.ts @@ -1,5 +1,5 @@ -import type { Psql } from "@core/db/psql.d.ts"; -import { BiliVideoMetadataType, BiliUserType } from "@core/db/schema"; +import type { Psql } from "@core/db/psql.d"; +import { BiliUserType, BiliVideoMetadataType } from "@core/db/schema"; import { AkariModelVersion } from "ml/const"; export async function videoExistsInAllData(sql: Psql, aid: number) { @@ -30,7 +30,7 @@ export async function insertVideoLabel(sql: Psql, aid: number, label: number) { } export async function getVideoInfoFromAllData(sql: Psql, aid: number) { - const rows = await sql` + const rows = await sql` SELECT * FROM bilibili_metadata WHERE aid = ${aid} `; const row = rows[0]; @@ -52,16 +52,6 @@ export async function getVideoInfoFromAllData(sql: Psql, aid: number) { }; } -export async function getUnArchivedBiliUsers(sql: Psql) { - const rows = await sql<{ uid: number }[]>` - SELECT ad.uid - FROM bilibili_metadata ad - LEFT JOIN bilibili_user bu ON ad.uid = bu.uid - WHERE bu.uid IS NULL; - `; - return rows.map((row) => row.uid); -} - export async function setBiliVideoStatus(sql: Psql, aid: number, status: number) { await sql` UPDATE bilibili_metadata SET status = ${status} WHERE aid = ${aid} diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index e2c8555..1241be0 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -1,6 +1,6 @@ import { LatestSnapshotType } from "@core/db/schema"; -import { SnapshotNumber } from "mq/task/getVideoStats.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { SnapshotNumber } from "mq/task/getVideoStats"; +import type { Psql } from "@core/db/psql.d"; export async function getVideosNearMilestone(sql: Psql) { const queryResult = await sql` diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index 7def56c..0f4223e 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -1,10 +1,10 @@ -import type { SnapshotScheduleType } from "@core/db/schema.d.ts"; -import logger from "@core/log/logger.ts"; -import { MINUTE } from "@core/const/time.ts"; -import { redis } from "@core/db/redis.ts"; +import type { SnapshotScheduleType } from "@core/db/schema.d"; +import logger from "@core/log"; +import { MINUTE } from "@core/lib"; +import { redis } from "@core/db/redis"; import { Redis } from "ioredis"; -import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre"; +import type { Psql } from "@core/db/psql.d"; const REDIS_KEY = "cvsa:snapshot_window_counts"; diff --git a/packages/crawler/db/songs.ts b/packages/crawler/db/songs.ts index ee4f042..413e683 100644 --- a/packages/crawler/db/songs.ts +++ b/packages/crawler/db/songs.ts @@ -1,5 +1,5 @@ -import type { Psql } from "@core/db/psql.d.ts"; -import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts"; +import type { Psql } from "@core/db/psql.d"; +import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre"; export async function getNotCollectedSongs(sql: Psql) { const rows = await sql<{ aid: number }[]>` diff --git a/packages/crawler/main.ts b/packages/crawler/main.ts deleted file mode 100644 index 7f21c20..0000000 --- a/packages/crawler/main.ts +++ /dev/null @@ -1,7 +0,0 @@ -// DENO ASK ME TO EXPORT SOMETHING WHEN 'name' IS SPECIFIED -// AND IF I DON'T SPECIFY 'name', THE --filter FLAG IN `deno task` WON'T WORK. -// I DONT'T KNOW WHY -// SO HERE'S A PLACHOLDER EXPORT FOR DENO: -export const DENO = "FUCK YOU DENO"; -// Oh, maybe export the version is a good idea -export const VERSION = "1.0.26"; diff --git a/packages/crawler/ml/akari.ts b/packages/crawler/ml/akari.ts index ebf085f..6f7f83f 100644 --- a/packages/crawler/ml/akari.ts +++ b/packages/crawler/ml/akari.ts @@ -1,7 +1,7 @@ -import { AIManager } from "ml/manager.ts"; +import { AIManager } from "ml/manager"; import * as ort from "onnxruntime-node"; -import logger from "@core/log/logger.ts"; -import { WorkerError } from "mq/schema.ts"; +import logger from "@core/log"; +import { WorkerError } from "mq/schema"; import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers"; import { AkariModelVersion } from "./const"; @@ -91,10 +91,6 @@ class AkariProto extends AIManager { } return probabilities.indexOf(Math.max(...probabilities)); } - - public getModelVersion(): string { - return this.modelVersion; - } } const Akari = new AkariProto(); diff --git a/packages/crawler/ml/manager.ts b/packages/crawler/ml/manager.ts index d193bc6..eefcbaa 100644 --- a/packages/crawler/ml/manager.ts +++ b/packages/crawler/ml/manager.ts @@ -1,6 +1,6 @@ import * as ort from "onnxruntime-node"; -import logger from "@core/log/logger.ts"; -import { WorkerError } from "mq/schema.ts"; +import logger from "@core/log"; +import { WorkerError } from "mq/schema"; export class AIManager { public sessions: { [key: string]: ort.InferenceSession } = {}; diff --git a/packages/crawler/mq/exec/archiveSnapshots.ts b/packages/crawler/mq/exec/archiveSnapshots.ts index 3110141..158ba4b 100644 --- a/packages/crawler/mq/exec/archiveSnapshots.ts +++ b/packages/crawler/mq/exec/archiveSnapshots.ts @@ -1,9 +1,9 @@ import { Job } from "bullmq"; -import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts"; -import logger from "@core/log/logger.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; -import { getLatestVideoSnapshot } from "db/snapshot.ts"; -import { MINUTE } from "@core/const/time.ts"; +import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule"; +import logger from "@core/log"; +import { lockManager } from "@core/mq/lockManager"; +import { getLatestVideoSnapshot } from "db/snapshot"; +import { MINUTE } from "@core/lib"; import { sql } from "@core/db/dbNew"; function getNextSaturdayMidnightTimestamp(): number { diff --git a/packages/crawler/mq/exec/classifyVideo.ts b/packages/crawler/mq/exec/classifyVideo.ts index 50a760b..04b52e2 100644 --- a/packages/crawler/mq/exec/classifyVideo.ts +++ b/packages/crawler/mq/exec/classifyVideo.ts @@ -1,14 +1,14 @@ import { Job } from "bullmq"; -import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "../../db/bilibili_metadata.ts"; -import Akari from "ml/akari.ts"; -import { ClassifyVideoQueue } from "mq/index.ts"; -import logger from "@core/log/logger.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; -import { aidExistsInSongs } from "db/songs.ts"; -import { insertIntoSongs } from "mq/task/collectSongs.ts"; -import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import { MINUTE } from "@core/const/time.ts"; -import { sql } from "@core/db/dbNew.ts"; +import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "../../db/bilibili_metadata"; +import Akari from "ml/akari"; +import { ClassifyVideoQueue } from "mq/index"; +import logger from "@core/log"; +import { lockManager } from "@core/mq/lockManager"; +import { aidExistsInSongs } from "db/songs"; +import { insertIntoSongs } from "mq/task/collectSongs"; +import { scheduleSnapshot } from "db/snapshotSchedule"; +import { MINUTE } from "@core/lib"; +import { sql } from "@core/db/dbNew"; export const classifyVideoWorker = async (job: Job) => { const aid = job.data.aid; diff --git a/packages/crawler/mq/exec/collectSongs.ts b/packages/crawler/mq/exec/collectSongs.ts index b354c2b..02f2bc6 100644 --- a/packages/crawler/mq/exec/collectSongs.ts +++ b/packages/crawler/mq/exec/collectSongs.ts @@ -1,5 +1,5 @@ import { Job } from "bullmq"; -import { collectSongs } from "mq/task/collectSongs.ts"; +import { collectSongs } from "mq/task/collectSongs"; export const collectSongsWorker = async (_job: Job): Promise => { await collectSongs(); diff --git a/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts b/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts index 1ed8b26..0022e02 100644 --- a/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts @@ -1,10 +1,10 @@ import { Job } from "bullmq"; -import { getVideosNearMilestone } from "db/snapshot.ts"; -import { getAdjustedShortTermETA } from "mq/scheduling.ts"; -import { truncate } from "utils/truncate.ts"; -import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import logger from "@core/log/logger.ts"; -import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; +import { getVideosNearMilestone } from "db/snapshot"; +import { getAdjustedShortTermETA } from "mq/scheduling"; +import { truncate } from "utils/truncate"; +import { scheduleSnapshot } from "db/snapshotSchedule"; +import logger from "@core/log"; +import { HOUR, MINUTE, SECOND } from "@core/lib"; import { sql } from "@core/db/dbNew"; export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => { diff --git a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts index 9b47b03..d0ef8ba 100644 --- a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts @@ -1,12 +1,12 @@ import { Job } from "bullmq"; -import { getLatestVideoSnapshot } from "db/snapshot.ts"; -import { truncate } from "utils/truncate.ts"; -import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule.ts"; -import logger from "@core/log/logger.ts"; -import { HOUR, MINUTE, WEEK } from "@core/const/time.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; -import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts"; -import { sql } from "@core/db/dbNew.ts"; +import { getLatestVideoSnapshot } from "db/snapshot"; +import { truncate } from "utils/truncate"; +import { getVideosWithoutActiveSnapshotScheduleByType, scheduleSnapshot } from "db/snapshotSchedule"; +import logger from "@core/log"; +import { HOUR, MINUTE, WEEK } from "@core/lib"; +import { lockManager } from "@core/mq/lockManager"; +import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval"; +import { sql } from "@core/db/dbNew"; export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise => { try { diff --git a/packages/crawler/mq/exec/executors.ts b/packages/crawler/mq/exec/executors.ts index c2969ed..ddeaa4c 100644 --- a/packages/crawler/mq/exec/executors.ts +++ b/packages/crawler/mq/exec/executors.ts @@ -1,10 +1,10 @@ -export * from "mq/exec/getLatestVideos.ts"; -export * from "./getVideoInfo.ts"; -export * from "./collectSongs.ts"; -export * from "./takeBulkSnapshot.ts"; -export * from "./archiveSnapshots.ts"; -export * from "./dispatchMilestoneSnapshots.ts"; -export * from "./dispatchRegularSnapshots.ts"; -export * from "./snapshotVideo.ts"; -export * from "./scheduleCleanup.ts"; -export * from "./snapshotTick.ts"; +export * from "./getLatestVideos"; +export * from "./getVideoInfo"; +export * from "./collectSongs"; +export * from "./takeBulkSnapshot"; +export * from "./archiveSnapshots"; +export * from "./dispatchMilestoneSnapshots"; +export * from "./dispatchRegularSnapshots"; +export * from "./snapshotVideo"; +export * from "./scheduleCleanup"; +export * from "./snapshotTick"; diff --git a/packages/crawler/mq/exec/getLatestVideos.ts b/packages/crawler/mq/exec/getLatestVideos.ts index f2f4351..85a72e1 100644 --- a/packages/crawler/mq/exec/getLatestVideos.ts +++ b/packages/crawler/mq/exec/getLatestVideos.ts @@ -1,6 +1,6 @@ import { sql } from "@core/db/dbNew"; import { Job } from "bullmq"; -import { queueLatestVideos } from "mq/task/queueLatestVideo.ts"; +import { queueLatestVideos } from "mq/task/queueLatestVideo"; export const getLatestVideosWorker = async (_job: Job): Promise => { await queueLatestVideos(sql); diff --git a/packages/crawler/mq/exec/getVideoInfo.ts b/packages/crawler/mq/exec/getVideoInfo.ts index 889bd7e..4ff50d3 100644 --- a/packages/crawler/mq/exec/getVideoInfo.ts +++ b/packages/crawler/mq/exec/getVideoInfo.ts @@ -1,6 +1,6 @@ import { Job } from "bullmq"; -import { insertVideoInfo } from "mq/task/getVideoDetails.ts"; -import logger from "@core/log/logger.ts"; +import { insertVideoInfo } from "mq/task/getVideoDetails"; +import logger from "@core/log"; import { sql } from "@core/db/dbNew"; export const getVideoInfoWorker = async (job: Job): Promise => { diff --git a/packages/crawler/mq/exec/scheduleCleanup.ts b/packages/crawler/mq/exec/scheduleCleanup.ts index 821abe4..4f9a664 100644 --- a/packages/crawler/mq/exec/scheduleCleanup.ts +++ b/packages/crawler/mq/exec/scheduleCleanup.ts @@ -1,10 +1,10 @@ import { Job } from "bullmq"; import { sql } from "@core/db/dbNew"; -import logger from "@core/log/logger.ts"; -import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule.ts"; -import { SECOND } from "@core/const/time.ts"; -import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount.ts"; -import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules.ts"; +import logger from "@core/log"; +import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule"; +import { SECOND } from "@core/lib"; +import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount"; +import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules"; interface SnapshotSchedule { id: bigint; diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts index 784bb04..1bc1446 100644 --- a/packages/crawler/mq/exec/snapshotTick.ts +++ b/packages/crawler/mq/exec/snapshotTick.ts @@ -6,9 +6,9 @@ import { getSnapshotsInNextSecond, setSnapshotStatus, videoHasProcessingSchedule -} from "db/snapshotSchedule.ts"; -import logger from "@core/log/logger.ts"; -import { SnapshotQueue } from "mq/index.ts"; +} from "db/snapshotSchedule"; +import logger from "@core/log"; +import { SnapshotQueue } from "mq/index"; import { sql } from "@core/db/dbNew"; const priorityMap: { [key: string]: number } = { diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts index 9f806d5..eaecf04 100644 --- a/packages/crawler/mq/exec/snapshotVideo.ts +++ b/packages/crawler/mq/exec/snapshotVideo.ts @@ -1,14 +1,14 @@ import { Job } from "bullmq"; -import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts"; -import logger from "@core/log/logger.ts"; -import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; -import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts"; -import { insertVideoSnapshot } from "mq/task/getVideoStats.ts"; -import { getSongsPublihsedAt } from "db/songs.ts"; -import { getAdjustedShortTermETA } from "mq/scheduling.ts"; -import { NetSchedulerError } from "@core/net/delegate.ts"; -import { sql } from "@core/db/dbNew.ts"; -import { closetMilestone } from "./snapshotTick.ts"; +import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule"; +import logger from "@core/log"; +import { HOUR, MINUTE, SECOND } from "@core/lib"; +import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata"; +import { insertVideoSnapshot } from "mq/task/getVideoStats"; +import { getSongsPublihsedAt } from "db/songs"; +import { getAdjustedShortTermETA } from "mq/scheduling"; +import { NetSchedulerError } from "@core/net/delegate"; +import { sql } from "@core/db/dbNew"; +import { closetMilestone } from "./snapshotTick"; const snapshotTypeToTaskMap: { [key: string]: string } = { milestone: "snapshotMilestoneVideo", diff --git a/packages/crawler/mq/exec/takeBulkSnapshot.ts b/packages/crawler/mq/exec/takeBulkSnapshot.ts index 00a9354..7b7ca9b 100644 --- a/packages/crawler/mq/exec/takeBulkSnapshot.ts +++ b/packages/crawler/mq/exec/takeBulkSnapshot.ts @@ -4,14 +4,14 @@ import { bulkSetSnapshotStatus, scheduleSnapshot, snapshotScheduleExists -} from "db/snapshotSchedule.ts"; -import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts"; -import logger from "@core/log/logger.ts"; -import { NetSchedulerError } from "@core/net/delegate.ts"; -import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; -import { getRegularSnapshotInterval } from "../task/regularSnapshotInterval.ts"; +} from "db/snapshotSchedule"; +import { bulkGetVideoStats } from "net/bulkGetVideoStats"; +import logger from "@core/log"; +import { NetSchedulerError } from "@core/net/delegate"; +import { HOUR, MINUTE, SECOND } from "@core/lib"; +import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval"; import { SnapshotScheduleType } from "@core/db/schema"; -import { sql } from "@core/db/dbNew.ts"; +import { sql } from "@core/db/dbNew"; export const takeBulkSnapshotForVideosWorker = async (job: Job) => { const schedules: SnapshotScheduleType[] = job.data.schedules; diff --git a/packages/crawler/mq/index.ts b/packages/crawler/mq/index.ts index 0644d05..869e716 100644 --- a/packages/crawler/mq/index.ts +++ b/packages/crawler/mq/index.ts @@ -1,5 +1,5 @@ import { Queue, ConnectionOptions } from "bullmq"; -import { redis } from "@core/db/redis.ts"; +import { redis } from "@core/db/redis"; export const LatestVideosQueue = new Queue("latestVideos", { connection: redis as ConnectionOptions diff --git a/packages/crawler/mq/init.ts b/packages/crawler/mq/init.ts index e6c5e52..fb94ed8 100644 --- a/packages/crawler/mq/init.ts +++ b/packages/crawler/mq/init.ts @@ -1,8 +1,8 @@ -import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; -import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.ts"; -import logger from "@core/log/logger.ts"; -import { initSnapshotWindowCounts } from "db/snapshotSchedule.ts"; -import { redis } from "@core/db/redis.ts"; +import { HOUR, MINUTE, SECOND } from "@core/lib"; +import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index"; +import logger from "@core/log"; +import { initSnapshotWindowCounts } from "db/snapshotSchedule"; +import { redis } from "@core/db/redis"; import { sql } from "@core/db/dbNew"; export async function initMQ() { diff --git a/packages/crawler/mq/scheduling.ts b/packages/crawler/mq/scheduling.ts index dcc8ad3..8012749 100644 --- a/packages/crawler/mq/scheduling.ts +++ b/packages/crawler/mq/scheduling.ts @@ -1,8 +1,8 @@ -import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts"; -import { truncate } from "utils/truncate.ts"; -import { closetMilestone } from "./exec/snapshotTick.ts"; -import { HOUR, MINUTE } from "@core/const/time.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule"; +import { truncate } from "utils/truncate"; +import { closetMilestone } from "./exec/snapshotTick"; +import { HOUR, MINUTE } from "@core/lib"; +import type { Psql } from "@core/db/psql.d"; const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); diff --git a/packages/crawler/mq/task/collectSongs.ts b/packages/crawler/mq/task/collectSongs.ts index af3178e..f5eb238 100644 --- a/packages/crawler/mq/task/collectSongs.ts +++ b/packages/crawler/mq/task/collectSongs.ts @@ -1,9 +1,9 @@ import { sql } from "@core/db/dbNew"; -import { aidExistsInSongs, getNotCollectedSongs } from "db/songs.ts"; -import logger from "@core/log/logger.ts"; -import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import { MINUTE } from "@core/const/time.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { aidExistsInSongs, getNotCollectedSongs } from "db/songs"; +import logger from "@core/log"; +import { scheduleSnapshot } from "db/snapshotSchedule"; +import { MINUTE } from "@core/lib"; +import type { Psql } from "@core/db/psql.d"; export async function collectSongs() { const aids = await getNotCollectedSongs(sql); diff --git a/packages/crawler/mq/task/getVideoDetails.ts b/packages/crawler/mq/task/getVideoDetails.ts index cd67994..5fb0b65 100644 --- a/packages/crawler/mq/task/getVideoDetails.ts +++ b/packages/crawler/mq/task/getVideoDetails.ts @@ -1,10 +1,10 @@ -import { getVideoDetails } from "net/getVideoDetails.ts"; -import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts"; -import logger from "@core/log/logger.ts"; -import { ClassifyVideoQueue } from "mq/index.ts"; -import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata.ts"; -import { HOUR, SECOND } from "@core/const/time.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { getVideoDetails } from "net/getVideoDetails"; +import { formatTimestampToPsql } from "utils/formatTimestampToPostgre"; +import logger from "@core/log"; +import { ClassifyVideoQueue } from "mq/index"; +import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata"; +import { HOUR, SECOND } from "@core/lib"; +import type { Psql } from "@core/db/psql.d"; export async function insertVideoInfo(sql: Psql, aid: number) { const videoExists = await videoExistsInAllData(sql, aid); diff --git a/packages/crawler/mq/task/getVideoStats.ts b/packages/crawler/mq/task/getVideoStats.ts index afd5b72..20b068d 100644 --- a/packages/crawler/mq/task/getVideoStats.ts +++ b/packages/crawler/mq/task/getVideoStats.ts @@ -1,6 +1,6 @@ -import { getVideoInfo } from "@core/net/getVideoInfo.ts"; -import logger from "@core/log/logger.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { getVideoInfo } from "@core/net/getVideoInfo"; +import logger from "@core/log"; +import type { Psql } from "@core/db/psql.d"; export interface SnapshotNumber { time: number; diff --git a/packages/crawler/mq/task/queueLatestVideo.ts b/packages/crawler/mq/task/queueLatestVideo.ts index 193787a..45d0e02 100644 --- a/packages/crawler/mq/task/queueLatestVideo.ts +++ b/packages/crawler/mq/task/queueLatestVideo.ts @@ -1,10 +1,10 @@ -import { getLatestVideoAids } from "net/getLatestVideoAids.ts"; -import { videoExistsInAllData } from "db/bilibili_metadata.ts"; -import { sleep } from "utils/sleep.ts"; -import { SECOND } from "@core/const/time.ts"; -import logger from "@core/log/logger.ts"; -import { LatestVideosQueue } from "mq/index.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { getLatestVideoAids } from "net/getLatestVideoAids"; +import { videoExistsInAllData } from "db/bilibili_metadata"; +import { sleep } from "utils/sleep"; +import { SECOND } from "@core/lib"; +import logger from "@core/log"; +import { LatestVideosQueue } from "mq/index"; +import type { Psql } from "@core/db/psql.d"; export async function queueLatestVideos(sql: Psql): Promise { let page = 1; diff --git a/packages/crawler/mq/task/regularSnapshotInterval.ts b/packages/crawler/mq/task/regularSnapshotInterval.ts index 306865a..40d2b6a 100644 --- a/packages/crawler/mq/task/regularSnapshotInterval.ts +++ b/packages/crawler/mq/task/regularSnapshotInterval.ts @@ -1,6 +1,6 @@ -import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts"; -import { HOUR } from "@core/const/time.ts"; -import type { Psql } from "@core/db/psql.d.ts"; +import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule"; +import { HOUR } from "@core/lib"; +import type { Psql } from "@core/db/psql.d"; export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => { const now = Date.now(); diff --git a/packages/crawler/mq/task/removeAllTimeoutSchedules.ts b/packages/crawler/mq/task/removeAllTimeoutSchedules.ts index 325f08b..912ca5b 100644 --- a/packages/crawler/mq/task/removeAllTimeoutSchedules.ts +++ b/packages/crawler/mq/task/removeAllTimeoutSchedules.ts @@ -1,9 +1,9 @@ import { sql } from "@core/db/dbNew"; -import logger from "@core/log/logger.ts"; +import logger from "@core/log"; export async function removeAllTimeoutSchedules() { logger.log("Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker"); - return await sql` + return sql` DELETE FROM snapshot_schedule WHERE status IN ('pending', 'processing') AND started_at < NOW() - INTERVAL '30 minutes' diff --git a/packages/crawler/net/bulkGetVideoStats.ts b/packages/crawler/net/bulkGetVideoStats.ts index dede37f..049ec96 100644 --- a/packages/crawler/net/bulkGetVideoStats.ts +++ b/packages/crawler/net/bulkGetVideoStats.ts @@ -1,6 +1,6 @@ -import networkDelegate from "@core/net/delegate.ts"; -import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d.ts"; -import logger from "@core/log/logger.ts"; +import networkDelegate from "@core/net/delegate"; +import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d"; +import logger from "@core/log"; /* * Bulk fetch video metadata from bilibili API diff --git a/packages/crawler/net/getLatestVideoAids.ts b/packages/crawler/net/getLatestVideoAids.ts index f466653..12ba8dd 100644 --- a/packages/crawler/net/getLatestVideoAids.ts +++ b/packages/crawler/net/getLatestVideoAids.ts @@ -1,6 +1,6 @@ -import type { VideoListResponse } from "@core/net/bilibili.d.ts"; -import logger from "@core/log/logger.ts"; -import networkDelegate from "@core/net/delegate.ts"; +import type { VideoListResponse } from "@core/net/bilibili.d"; +import logger from "@core/log"; +import networkDelegate from "@core/net/delegate"; export async function getLatestVideoAids(page: number = 1, pageSize: number = 10): Promise { const startFrom = 1 + pageSize * (page - 1); diff --git a/packages/crawler/net/getVideoDetails.ts b/packages/crawler/net/getVideoDetails.ts index 432cbf2..3a6f515 100644 --- a/packages/crawler/net/getVideoDetails.ts +++ b/packages/crawler/net/getVideoDetails.ts @@ -1,6 +1,6 @@ -import networkDelegate from "@core/net/delegate.ts"; -import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d.ts"; -import logger from "@core/log/logger.ts"; +import networkDelegate from "@core/net/delegate"; +import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d"; +import logger from "@core/log"; export async function getVideoDetails(aid: number, archive: boolean = false): Promise { const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`; diff --git a/packages/crawler/src/bullui.ts b/packages/crawler/src/bullui.ts index 8f2dcc7..da86d71 100644 --- a/packages/crawler/src/bullui.ts +++ b/packages/crawler/src/bullui.ts @@ -2,7 +2,7 @@ import express from "express"; import { createBullBoard } from "@bull-board/api"; import { BullMQAdapter } from "@bull-board/api/bullMQAdapter.js"; import { ExpressAdapter } from "@bull-board/express"; -import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.ts"; +import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index"; const serverAdapter = new ExpressAdapter(); serverAdapter.setBasePath("/"); diff --git a/packages/crawler/src/filterWorker.ts b/packages/crawler/src/filterWorker.ts index c740336..a6b6d99 100644 --- a/packages/crawler/src/filterWorker.ts +++ b/packages/crawler/src/filterWorker.ts @@ -1,10 +1,10 @@ import { ConnectionOptions, Job, Worker } from "bullmq"; -import { redis } from "@core/db/redis.ts"; -import logger from "@core/log/logger.ts"; -import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts"; -import { WorkerError } from "mq/schema.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; -import Akari from "ml/akari.ts"; +import { redis } from "@core/db/redis"; +import logger from "@core/log"; +import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo"; +import { WorkerError } from "mq/schema"; +import { lockManager } from "@core/mq/lockManager"; +import Akari from "ml/akari"; const shutdown = async (signal: string) => { logger.log(`${signal} Received: Shutting down workers...`, "mq"); diff --git a/packages/crawler/src/jobAdder.ts b/packages/crawler/src/jobAdder.ts index 3aefd24..1a95f6f 100644 --- a/packages/crawler/src/jobAdder.ts +++ b/packages/crawler/src/jobAdder.ts @@ -1,3 +1,3 @@ -import { initMQ } from "mq/init.ts"; +import { initMQ } from "mq/init"; await initMQ(); diff --git a/packages/crawler/src/worker.ts b/packages/crawler/src/worker.ts index 418c343..7dc42d6 100644 --- a/packages/crawler/src/worker.ts +++ b/packages/crawler/src/worker.ts @@ -11,11 +11,11 @@ import { snapshotTickWorker, snapshotVideoWorker, takeBulkSnapshotForVideosWorker -} from "mq/exec/executors.ts"; -import { redis } from "@core/db/redis.ts"; -import logger from "@core/log/logger.ts"; -import { lockManager } from "@core/mq/lockManager.ts"; -import { WorkerError } from "mq/schema.ts"; +} from "mq/exec/executors"; +import { redis } from "@core/db/redis"; +import logger from "@core/log"; +import { lockManager } from "@core/mq/lockManager"; +import { WorkerError } from "mq/schema"; const releaseLockForJob = async (name: string) => { await lockManager.releaseLock(name); diff --git a/packages/crawler/test/db/snapshotSchedule.test.ts b/packages/crawler/test/db/snapshotSchedule.test.ts deleted file mode 100644 index 28de210..0000000 --- a/packages/crawler/test/db/snapshotSchedule.test.ts +++ /dev/null @@ -1,120 +0,0 @@ -import { expect, test } from "vitest"; -import { sqlTest as sql } from "@core/db/dbNew"; -import { SnapshotScheduleType } from "@core/db/schema"; -import { bulkSetSnapshotStatus } from "db/snapshotSchedule"; - -const mockSnapshotSchedules: SnapshotScheduleType[] = [ - { - id: 1, - aid: 1234567890, - type: "normal", - created_at: "2025-05-04T00:00:00.000Z", - started_at: "2025-05-04T06:00:00.000Z", - finished_at: "2025-05-04T06:03:27.000Z", - status: "completed" - }, - { - id: 2, - aid: 9876543210, - type: "archive", - created_at: "2025-05-03T12:00:00.000Z", - started_at: "2025-05-03T13:00:00.000Z", - finished_at: null, - status: "failed" - }, - { - id: 3, - aid: 1122334455, - type: "milestone", - created_at: "2025-05-01T08:00:00.000Z", - started_at: "2025-05-01T08:12:00.000Z", - finished_at: null, - status: "processing" - } -]; - -const databasePreparationQuery = ` - CREATE SEQUENCE "snapshot_schedule_id_seq" - START WITH 1 - INCREMENT BY 1 - MINVALUE 1 - MAXVALUE 9223372036854775807 - CACHE 1; - - CREATE TABLE "snapshot_schedule"( - "id" bigint DEFAULT nextval('snapshot_schedule_id_seq'::regclass) NOT NULL, - "aid" bigint NOT NULL, - "type" text, - "created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, - "started_at" timestamp with time zone, - "finished_at" timestamp with time zone, - "status" text DEFAULT 'pending'::text NOT NULL - ); - - CREATE INDEX idx_snapshot_schedule_aid ON snapshot_schedule USING btree (aid); - CREATE INDEX idx_snapshot_schedule_started_at ON snapshot_schedule USING btree (started_at); - CREATE INDEX idx_snapshot_schedule_status ON snapshot_schedule USING btree (status); - CREATE INDEX idx_snapshot_schedule_type ON snapshot_schedule USING btree (type); - CREATE UNIQUE INDEX snapshot_schedule_pkey ON snapshot_schedule USING btree (id); -`; - -const cleanUpQuery = ` - DROP SEQUENCE IF EXISTS "snapshot_schedule_id_seq" CASCADE; - DROP TABLE IF EXISTS "snapshot_schedule" CASCADE; -`; - -async function testMocking() { - await sql.begin(async (tx) => { - await tx.unsafe(cleanUpQuery).simple(); - await tx.unsafe(databasePreparationQuery).simple(); - - await tx` - INSERT INTO snapshot_schedule - ${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")} - `; - - await tx` - ROLLBACK; - `; - - await tx.unsafe(cleanUpQuery).simple(); - return; - }); -} - -async function testBulkSetSnapshotStatus() { - return await sql.begin(async (tx) => { - await tx.unsafe(cleanUpQuery).simple(); - await tx.unsafe(databasePreparationQuery).simple(); - - await tx` - INSERT INTO snapshot_schedule - ${sql(mockSnapshotSchedules, "aid", "created_at", "finished_at", "id", "started_at", "status", "type")} - `; - - const ids = [1, 2, 3]; - - await bulkSetSnapshotStatus(tx, ids, "pending"); - - const rows = tx<{ status: string }[]>` - SELECT status FROM snapshot_schedule WHERE id = 1; - `.execute(); - - await tx` - ROLLBACK; - `; - - await tx.unsafe(cleanUpQuery).simple(); - return rows; - }); -} - -test("data mocking works", async () => { - await testMocking(); - expect(() => {}).not.toThrowError(); -}); - -test("bulkSetSnapshotStatus core logic works smoothly", async () => { - const rows = await testBulkSetSnapshotStatus(); - expect(rows.every((item) => item.status === "pending")).toBe(true); -}); diff --git a/packages/crawler/tsconfig.json b/packages/crawler/tsconfig.json index 8ceed3a..5902fcd 100644 --- a/packages/crawler/tsconfig.json +++ b/packages/crawler/tsconfig.json @@ -2,17 +2,16 @@ "include": ["**/*.ts"], "compilerOptions": { "baseUrl": ".", - "target": "ESNext", - "module": "ESNext", + "target": "esnext", + "module": "esnext", "useDefineForClassFields": true, - "moduleResolution": "node", + "moduleResolution": "node10", "lib": ["ESNext", "DOM", "DOM.Iterable"], "skipLibCheck": true, "paths": { "@core/*": ["../core/*"] }, "allowSyntheticDefaultImports": true, - "allowImportingTsExtensions": true, "noEmit": true } } diff --git a/packages/crawler/vitest.config.ts b/packages/crawler/vitest.config.ts deleted file mode 100644 index 63af38b..0000000 --- a/packages/crawler/vitest.config.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { defineConfig } from "vitest/config"; -import tsconfigPaths from "vite-tsconfig-paths"; - -export default defineConfig({ - plugins: [tsconfigPaths()] -}); diff --git a/packages/elysia/routes/song/milestone.ts b/packages/elysia/routes/song/milestone.ts new file mode 100644 index 0000000..f944fcb --- /dev/null +++ b/packages/elysia/routes/song/milestone.ts @@ -0,0 +1,41 @@ +import { Elysia, t } from "elysia"; +import { dbMain } from "@core/drizzle"; +import { bilibiliMetadata, latestVideoSnapshot } from "@core/drizzle/main/schema"; +import { eq, and, gte, lt, desc } from "drizzle-orm"; + +type MileStoneType = "dendou" | "densetsu" | "shinwa"; + +const range = { + dendou: [90000, 99999], + densetsu: [900000, 999999], + shinwa: [5000000, 9999999] +}; + +export const closeMileStoneHandler = new Elysia({ prefix: "/song" }).get( + "/close-milestone/:type", + async (c) => { + const type = c.params.type; + const min = range[type as MileStoneType][0]; + const max = range[type as MileStoneType][1]; + const data = await dbMain + .select() + .from(bilibiliMetadata) + .innerJoin(latestVideoSnapshot, eq(latestVideoSnapshot.aid, bilibiliMetadata.aid)) + .where(and(gte(latestVideoSnapshot.views, min), lt(latestVideoSnapshot.views, max))) + .orderBy(desc(latestVideoSnapshot.views)); + const aids = data.map((song) => song.bilibili_metadata.aid); + for (const aid of aids) { + + } + + return data; + }, + { + response: { + 200: t.Array(t.Any()), + 404: t.Object({ + message: t.String() + }) + } + } +); diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index 30feae7..b855b71 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -15,9 +15,7 @@ const app = new Elysia({ hostname: host } }) - // @ts-expect-error idfk .use(cors()) - // @ts-expect-error idfk .use(openapi()) .use(rootHandler) .use(pingHandler) diff --git a/packages/next/lib/net.ts b/packages/next/lib/net.ts index 546b887..1ac7f2e 100644 --- a/packages/next/lib/net.ts +++ b/packages/next/lib/net.ts @@ -1,4 +1,4 @@ -import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios"; +import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, Method } from "axios"; export class ApiRequestError extends Error { public code: number | undefined; @@ -49,8 +49,7 @@ export async function fetcher( const response = await httpMethods[m](url, data, fullConfig); return response.data; } else if (m === "delete") { - const response = await axios.delete(url, fullConfig); - return response; + return await axios.delete(url, fullConfig); } else { const response = await httpMethods[m](url, fullConfig); return response.data; diff --git a/packages/solid/src/lib/net.ts b/packages/solid/src/lib/net.ts index 546b887..1ac7f2e 100644 --- a/packages/solid/src/lib/net.ts +++ b/packages/solid/src/lib/net.ts @@ -1,4 +1,4 @@ -import axios, { AxiosRequestConfig, AxiosError, Method, AxiosResponse } from "axios"; +import axios, { AxiosError, AxiosRequestConfig, AxiosResponse, Method } from "axios"; export class ApiRequestError extends Error { public code: number | undefined; @@ -49,8 +49,7 @@ export async function fetcher( const response = await httpMethods[m](url, data, fullConfig); return response.data; } else if (m === "delete") { - const response = await axios.delete(url, fullConfig); - return response; + return await axios.delete(url, fullConfig); } else { const response = await httpMethods[m](url, fullConfig); return response.data; diff --git a/packages/temp_frontend/app/routes/home.tsx b/packages/temp_frontend/app/routes/home.tsx index 5de738f..1f4c6ae 100644 --- a/packages/temp_frontend/app/routes/home.tsx +++ b/packages/temp_frontend/app/routes/home.tsx @@ -1,7 +1,6 @@ import type { Route } from "./+types/home"; import { treaty } from "@elysiajs/eden"; import type { App } from "@elysia/src"; -import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { SearchIcon } from "@/components/icons/search"; diff --git a/src/fillSongInfo.ts b/src/fillSongInfo.ts index 313c963..4592d1e 100644 --- a/src/fillSongInfo.ts +++ b/src/fillSongInfo.ts @@ -1,5 +1,5 @@ import arg from "arg"; -import logger from "@core/log/logger"; +import logger from "@core/log"; import { sql } from "@core/index"; import type { Row } from "postgres"; diff --git a/src/fixCover.ts b/src/fixCover.ts index e95942c..0c05b92 100644 --- a/src/fixCover.ts +++ b/src/fixCover.ts @@ -1,6 +1,6 @@ import arg from "arg"; import { Database } from "bun:sqlite"; -import logger from "@core/log/logger"; +import logger from "@core/log"; import type { VideoDetailsData } from "@core/net/bilibili.d.ts"; import { sql } from "@core/index"; diff --git a/src/fixPubDate.ts b/src/fixPubDate.ts index 98c90d7..295a016 100644 --- a/src/fixPubDate.ts +++ b/src/fixPubDate.ts @@ -1,6 +1,6 @@ import arg from "arg"; import { Database } from "bun:sqlite"; -import logger from "@core/log/logger"; +import logger from "@core/log"; import type { VideoDetailsData } from "@core/net/bilibili.d.ts"; import { sql } from "@core/index";