Compare commits

...

17 Commits

Author SHA1 Message Date
9d5c7cc47d
update: new delegate that fits to new infra 2025-05-02 20:14:05 +08:00
7528dcdf81
debug: add debug logging in alicloudFcRequest 2025-05-02 20:01:21 +08:00
811a8261b3
fix: BullMQ crashes when not setting Redis config maxRetriesPerRequest to null 2025-05-02 19:48:17 +08:00
d0aa27b2ad
ref: backend structure, make backend & frontend containerize
fix: some errors in dockerfile
2025-04-29 21:45:38 +08:00
280825cf67
fix: missing dependency in frontend 2025-04-29 06:42:24 +08:00
e658135a81
fix: missing name in packages/core/package.json 2025-04-29 06:23:11 +08:00
4632bd5906
fix: missing dependency that causes failure of resolving import 2025-04-29 06:20:22 +08:00
43b52dee0b
add: complete docker config 2025-04-29 06:12:44 +08:00
784939074a
merge: branch 'ref/deno' into ref/docker 2025-04-29 04:00:57 +08:00
94dd662c40
fix: module not found "net/getVideoInfo.ts" 2025-04-29 04:00:15 +08:00
1ebc0d0c1b
update: .gitignore 2025-04-29 03:52:20 +08:00
728a74f4d3
merge: branch 'feat/backend' into ref/docker 2025-04-29 03:49:27 +08:00
0c8a459d92
update: .gitignore 2025-04-29 03:49:17 +08:00
d7b6792d05
add: basic docker config 2025-04-29 03:45:43 +08:00
50bcb48bd6
fix: incorrect key causing RangeError in date-fns 2025-04-29 00:33:12 +08:00
fe386d2b02
debug: add logs for info.astro 2025-04-29 00:24:05 +08:00
e72b3008d1
ref: switch to bun & postgres.js for backend 2025-04-29 00:07:12 +08:00
68 changed files with 2892 additions and 476 deletions

96
.dockerignore Normal file
View File

@ -0,0 +1,96 @@
built/*
tests/cases/rwc/*
tests/cases/perf/*
!tests/cases/webharness/compilerToString.js
test-args.txt
~*.docx
\#*\#
.\#*
tests/baselines/local/*
tests/baselines/local.old/*
tests/services/baselines/local/*
tests/baselines/prototyping/local/*
tests/baselines/rwc/*
tests/baselines/reference/projectOutput/*
tests/baselines/local/projectOutput/*
tests/baselines/reference/testresults.tap
tests/baselines/symlinks/*
tests/services/baselines/prototyping/local/*
tests/services/browser/typescriptServices.js
src/harness/*.js
src/compiler/diagnosticInformationMap.generated.ts
src/compiler/diagnosticMessages.generated.json
src/parser/diagnosticInformationMap.generated.ts
src/parser/diagnosticMessages.generated.json
rwc-report.html
*.swp
build.json
*.actual
tests/webTestServer.js
tests/webTestServer.js.map
tests/webhost/*.d.ts
tests/webhost/webtsc.js
tests/cases/**/*.js
tests/cases/**/*.js.map
*.config
scripts/eslint/built/
scripts/debug.bat
scripts/run.bat
scripts/**/*.js
scripts/**/*.js.map
coverage/
internal/
**/.DS_Store
.settings
**/.vs
**/.vscode/*
!**/.vscode/tasks.json
!**/.vscode/settings.template.json
!**/.vscode/launch.template.json
!**/.vscode/extensions.json
!tests/cases/projects/projectOption/**/node_modules
!tests/cases/projects/NodeModulesSearch/**/*
!tests/baselines/reference/project/nodeModules*/**/*
yarn.lock
yarn-error.log
.parallelperf.*
tests/baselines/reference/dt
.failed-tests
TEST-results.xml
package-lock.json
.eslintcache
*v8.log
# dotenv environment variable files
.env
.env.development.local
.env.test.local
.env.production.local
.env.local
# npm dependencies
node_modules/
# project specific
logs/
__pycache__
ml/filter/runs
ml/pred/runs
ml/pred/checkpoints
ml/pred/observed
ml/data/
ml/filter/checkpoints
scripts
model/
.astro
# Database
*.dump
*.db
*.sqlite
*.sqlite3
data/
docker-compose.yml

10
.gitignore vendored
View File

@ -85,3 +85,13 @@ model/
.astro
# Database
*.dump
*.db
*.sqlite
*.sqlite3
data/
redis/
docker-compose.yml

6
.idea/bun.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="BunSettings">
<option name="bunPath" value="$USER_HOME$/.bun/bin/bun" />
</component>
</project>

View File

@ -0,0 +1,55 @@
<component name="ProjectCodeStyleConfiguration">
<code_scheme name="Project" version="173">
<option name="LINE_SEPARATOR" value="&#10;" />
<HTMLCodeStyleSettings>
<option name="HTML_SPACE_INSIDE_EMPTY_TAG" value="true" />
</HTMLCodeStyleSettings>
<JSCodeStyleSettings version="0">
<option name="FORCE_SEMICOLON_STYLE" value="true" />
<option name="SPACE_BEFORE_FUNCTION_LEFT_PARENTH" value="false" />
<option name="FORCE_QUOTE_STYlE" value="true" />
<option name="ENFORCE_TRAILING_COMMA" value="Remove" />
<option name="SPACES_WITHIN_OBJECT_LITERAL_BRACES" value="true" />
<option name="SPACES_WITHIN_IMPORTS" value="true" />
</JSCodeStyleSettings>
<TypeScriptCodeStyleSettings version="0">
<option name="FORCE_SEMICOLON_STYLE" value="true" />
<option name="SPACE_BEFORE_FUNCTION_LEFT_PARENTH" value="false" />
<option name="FORCE_QUOTE_STYlE" value="true" />
<option name="ENFORCE_TRAILING_COMMA" value="Remove" />
<option name="SPACES_WITHIN_OBJECT_LITERAL_BRACES" value="true" />
<option name="SPACES_WITHIN_IMPORTS" value="true" />
</TypeScriptCodeStyleSettings>
<VueCodeStyleSettings>
<option name="INTERPOLATION_NEW_LINE_AFTER_START_DELIMITER" value="false" />
<option name="INTERPOLATION_NEW_LINE_BEFORE_END_DELIMITER" value="false" />
</VueCodeStyleSettings>
<codeStyleSettings language="HTML">
<option name="SOFT_MARGINS" value="120" />
<indentOptions>
<option name="CONTINUATION_INDENT_SIZE" value="4" />
<option name="USE_TAB_CHARACTER" value="true" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="JavaScript">
<option name="SOFT_MARGINS" value="120" />
<indentOptions>
<option name="USE_TAB_CHARACTER" value="true" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="TypeScript">
<option name="SOFT_MARGINS" value="120" />
<indentOptions>
<option name="USE_TAB_CHARACTER" value="true" />
</indentOptions>
</codeStyleSettings>
<codeStyleSettings language="Vue">
<option name="SOFT_MARGINS" value="120" />
<indentOptions>
<option name="INDENT_SIZE" value="4" />
<option name="TAB_SIZE" value="4" />
<option name="USE_TAB_CHARACTER" value="true" />
</indentOptions>
</codeStyleSettings>
</code_scheme>
</component>

View File

@ -0,0 +1,5 @@
<component name="ProjectCodeStyleConfiguration">
<state>
<option name="USE_PER_PROJECT_SETTINGS" value="true" />
</state>
</component>

6
.idea/compiler.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="TypeScriptCompiler">
<option name="useTypesFromServer" value="true" />
</component>
</project>

View File

@ -2,5 +2,6 @@
<project version="4">
<component name="DenoSettings">
<option name="denoInit" value="{&#10; &quot;enable&quot;: true,&#10; &quot;lint&quot;: true,&#10; &quot;unstable&quot;: true,&#10; &quot;importMap&quot;: &quot;import_map.json&quot;,&#10; &quot;config&quot;: &quot;deno.json&quot;,&#10; &quot;fmt&quot;: {&#10; &quot;useTabs&quot;: true,&#10; &quot;lineWidth&quot;: 120,&#10; &quot;indentWidth&quot;: 4,&#10; &quot;semiColons&quot;: true,&#10; &quot;proseWrap&quot;: &quot;always&quot;&#10; }&#10;}" />
<option name="useDenoValue" value="DISABLE" />
</component>
</project>

View File

@ -31,5 +31,6 @@
<option name="processLiterals" value="true" />
<option name="processComments" value="true" />
</inspection_tool>
<inspection_tool class="TsLint" enabled="true" level="WARNING" enabled_by_default="true" />
</profile>
</component>

17
Dockerfile.backend Normal file
View File

@ -0,0 +1,17 @@
FROM oven/bun AS bun-builder
WORKDIR /app
COPY ./packages/core ./core
COPY ./packages/backend/package.json ./packages/backend/bun.lock ./backend/
WORKDIR backend
RUN bun install
COPY ./packages/backend/ .
RUN mkdir -p /app/logs
CMD ["bun", "start"]

20
Dockerfile.frontend Normal file
View File

@ -0,0 +1,20 @@
FROM oven/bun AS bun-builder
WORKDIR /app
COPY . .
RUN bun install
WORKDIR packages/frontend
RUN bun run build
ENV HOST=0.0.0.0
ENV PORT=4321
EXPOSE 4321
RUN mkdir -p /app/logs
CMD ["bun", "/app/packages/frontend/dist/server/entry.mjs"]

1210
bun.lock Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,6 +1,6 @@
{
"lock": false,
"workspace": ["./packages/crawler", "./packages/frontend", "./packages/backend", "./packages/core"],
"workspace": ["./packages/crawler", "./packages/core"],
"nodeModulesDir": "auto",
"tasks": {
"crawler": "deno task --filter 'crawler' all",
@ -12,10 +12,5 @@
"indentWidth": 4,
"semiColons": true,
"proseWrap": "always"
},
"imports": {
"@astrojs/node": "npm:@astrojs/node@^9.1.3",
"@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8",
"date-fns": "npm:date-fns@^4.1.0"
}
}

14
package.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "cvsa",
"version": "2.13.22",
"private": false,
"type": "module",
"workspaces": [
"packages/frontend",
"packages/core",
"packages/backend"
],
"dependencies": {
"postgres": "^3.4.5"
}
}

View File

@ -0,0 +1,8 @@
{
"useTabs": true,
"tabWidth": 4,
"trailingComma": "none",
"singleQuote": false,
"printWidth": 120,
"endOfLine": "lf"
}

208
packages/backend/bun.lock Normal file
View File

@ -0,0 +1,208 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"dependencies": {
"@rabbit-company/argon2id": "^2.1.0",
"hono": "^4.7.8",
"hono-rate-limiter": "^0.4.2",
"ioredis": "^5.6.1",
"postgres": "^3.4.5",
"rate-limit-redis": "^4.2.0",
"yup": "^1.6.1",
"zod": "^3.24.3",
},
"devDependencies": {
"@types/bun": "^1.2.11",
"prettier": "^3.5.3",
},
},
},
"packages": {
"@ioredis/commands": ["@ioredis/commands@1.2.0", "", {}, "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="],
"@rabbit-company/argon2id": ["@rabbit-company/argon2id@2.1.0", "", { "peerDependencies": { "typescript": "^5.6.2" } }, "sha512-X/kt89qjmS9+Zh+DYCGcWeTwHa4C8vY8T3EnSma+vWj7spMzAYX4F8vmGUkny9hygpTOeC/yXwAUdJAfJ52H+w=="],
"@types/bun": ["@types/bun@1.2.11", "", { "dependencies": { "bun-types": "1.2.11" } }, "sha512-ZLbbI91EmmGwlWTRWuV6J19IUiUC5YQ3TCEuSHI3usIP75kuoA8/0PVF+LTrbEnVc8JIhpElWOxv1ocI1fJBbw=="],
"@types/node": ["@types/node@22.15.3", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-lX7HFZeHf4QG/J7tBZqrCAXwz9J5RD56Y6MpP0eJkka8p+K0RY/yBTW7CYFJ4VGCclxqOLKmiGP5juQc6MKgcw=="],
"accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="],
"body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="],
"bun-types": ["bun-types@1.2.11", "", { "dependencies": { "@types/node": "*" } }, "sha512-dbkp5Lo8HDrXkLrONm6bk+yiiYQSntvFUzQp0v3pzTAsXk6FtgVMjdQ+lzFNVAmQFUkPQZ3WMZqH5tTo+Dp/IA=="],
"bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="],
"call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="],
"call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="],
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
"content-disposition": ["content-disposition@1.0.0", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg=="],
"content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="],
"cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="],
"cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="],
"debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
"dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="],
"ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],
"encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
"es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
"es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
"es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="],
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
"express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="],
"express-rate-limit": ["express-rate-limit@7.5.0", "", { "peerDependencies": { "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg=="],
"finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="],
"forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="],
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
"get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="],
"get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="],
"gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
"has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="],
"hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
"hono": ["hono@4.7.8", "", {}, "sha512-PCibtFdxa7/Ldud9yddl1G81GjYaeMYYTq4ywSaNsYbB1Lug4mwtOMJf2WXykL0pntYwmpRJeOI3NmoDgD+Jxw=="],
"hono-rate-limiter": ["hono-rate-limiter@0.4.2", "", { "peerDependencies": { "hono": "^4.1.1" } }, "sha512-AAtFqgADyrmbDijcRTT/HJfwqfvhalya2Zo+MgfdrMPas3zSMD8SU03cv+ZsYwRU1swv7zgVt0shwN059yzhjw=="],
"http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
"iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
"ioredis": ["ioredis@5.6.1", "", { "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA=="],
"ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="],
"is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="],
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
"lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="],
"math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="],
"media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="],
"merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="],
"mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
"mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="],
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="],
"object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="],
"on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="],
"once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
"parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="],
"path-to-regexp": ["path-to-regexp@8.2.0", "", {}, "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ=="],
"postgres": ["postgres@3.4.5", "", {}, "sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg=="],
"prettier": ["prettier@3.5.3", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="],
"property-expr": ["property-expr@2.0.6", "", {}, "sha512-SVtmxhRE/CGkn3eZY1T6pC8Nln6Fr/lu1mKSgRud0eC73whjGfoAogbn78LkD8aFL0zz3bAFerKSnOl7NlErBA=="],
"proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="],
"qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="],
"range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
"rate-limit-redis": ["rate-limit-redis@4.2.0", "", { "peerDependencies": { "express-rate-limit": ">= 6" } }, "sha512-wV450NQyKC24NmPosJb2131RoczLdfIJdKCReNwtVpm5998U8SgKrAZrIHaN/NfQgqOHaan8Uq++B4sa5REwjA=="],
"raw-body": ["raw-body@3.0.0", "", { "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", "iconv-lite": "0.6.3", "unpipe": "1.0.0" } }, "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g=="],
"redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="],
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
"router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="],
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
"send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="],
"serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="],
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
"side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="],
"side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="],
"side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="],
"side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="],
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
"statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="],
"tiny-case": ["tiny-case@1.0.3", "", {}, "sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q=="],
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
"toposort": ["toposort@2.0.2", "", {}, "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg=="],
"type-fest": ["type-fest@2.19.0", "", {}, "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="],
"type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="],
"typescript": ["typescript@5.8.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ=="],
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
"unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="],
"vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="],
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
"yup": ["yup@1.6.1", "", { "dependencies": { "property-expr": "^2.0.5", "tiny-case": "^1.0.3", "toposort": "^2.0.2", "type-fest": "^2.19.0" } }, "sha512-JED8pB50qbA4FOkDol0bYF/p60qSEDQqBD0/qeIrUCG1KbPBIQ776fCUNb9ldbPcSTxA69g/47XTo4TqWiuXOA=="],
"zod": ["zod@3.24.3", "", {}, "sha512-HhY1oqzWCQWuUqvBFnsyrtZRhyPeR7SUGv+C4+MsisMuVfSPx8HpwWqH8tRahSlt6M3PiFAcoeFhZAqIXTxoSg=="],
}
}

View File

@ -1,30 +0,0 @@
import { type Client, Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { postgresConfig, postgresConfigCred } from "@core/db/pgConfig.ts";
import { createMiddleware } from "hono/factory";
const pool = new Pool(postgresConfig, 4);
const poolCred = new Pool(postgresConfigCred, 2);
export const db = pool;
export const dbCred = poolCred;
export const dbMiddleware = createMiddleware(async (c, next) => {
const connection = await pool.connect();
c.set("db", connection);
await next();
connection.release();
});
export const dbCredMiddleware = createMiddleware(async (c, next) => {
const connection = await poolCred.connect();
c.set("dbCred", connection);
await next();
connection.release();
});
declare module "hono" {
interface ContextVariableMap {
db: Client;
dbCred: Client;
}
}

View File

@ -0,0 +1,46 @@
const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT", "DB_NAME_CRED"];
const unsetVars = requiredEnvVars.filter((key) => process.env[key] === undefined);
if (unsetVars.length > 0) {
throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
}
const databaseHost = process.env["DB_HOST"]!;
const databaseName = process.env["DB_NAME"];
const databaseNameCred = process.env["DB_NAME_CRED"]!;
const databaseUser = process.env["DB_USER"]!;
const databasePassword = process.env["DB_PASSWORD"]!;
const databasePort = process.env["DB_PORT"]!;
export const postgresConfig = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseName,
user: databaseUser,
password: databasePassword
};
export const postgresConfigNpm = {
host: databaseHost,
port: parseInt(databasePort),
database: databaseName,
username: databaseUser,
password: databasePassword
};
export const postgresCredConfigNpm = {
host: databaseHost,
port: parseInt(databasePort),
database: databaseNameCred,
username: databaseUser,
password: databasePassword
}
export const postgresConfigCred = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseNameCred,
user: databaseUser,
password: databasePassword
};

View File

@ -0,0 +1,5 @@
import postgres from "postgres";
import { postgresConfigNpm, postgresCredConfigNpm } from "./config";
export const sql = postgres(postgresConfigNpm);
export const sqlCred = postgres(postgresCredConfigNpm)

View File

@ -1,21 +1,21 @@
import logger from "log/logger.ts";
import { Redis } from "ioredis";
import logger from "@core/log/logger.ts";
import { redis } from "@core/db/redis.ts";
import { sql } from "./db.ts";
import { number, ValidationError } from "yup";
import { createHandlers } from "./utils.ts";
import { getVideoInfo, getVideoInfoByBV } from "@crawler/net/videoInfo";
import { idSchema } from "./snapshots.ts";
import { createHandlers } from "../src/utils.ts";
import { getVideoInfo, getVideoInfoByBV } from "@core/net/getVideoInfo.ts";
import { idSchema } from "../routes/snapshots.ts";
import { NetSchedulerError } from "@core/net/delegate.ts";
import type { Context } from "hono";
import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import type { BlankEnv, BlankInput } from "hono/types";
import type { VideoInfoData } from "@core/net/bilibili.d.ts";
import { startTime, endTime } from 'hono/timing'
const redis = new Redis({ maxRetriesPerRequest: null });
const CACHE_EXPIRATION_SECONDS = 60;
type ContextType = Context<BlankEnv, "/video/:id/info", BlankInput>;
async function insertVideoSnapshot(client: Client, data: VideoInfoData) {
async function insertVideoSnapshot(data: VideoInfoData) {
const views = data.stat.view;
const danmakus = data.stat.danmaku;
const replies = data.stat.reply;
@ -25,21 +25,16 @@ async function insertVideoSnapshot(client: Client, data: VideoInfoData) {
const favorites = data.stat.favorite;
const aid = data.aid;
const query: string = `
await sql`
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
VALUES (${aid}, ${views}, ${danmakus}, ${replies}, ${likes}, ${coins}, ${shares}, ${favorites})
`;
await client.queryObject(
query,
[aid, views, danmakus, replies, likes, coins, shares, favorites],
);
logger.log(`Inserted into snapshot for video ${aid} by videoInfo API.`, "api", "fn:insertVideoSnapshot");
}
export const videoInfoHandler = createHandlers(async (c: ContextType) => {
const client = c.get("db");
startTime(c, 'parse', 'Parse the request');
try {
const id = await idSchema.validate(c.req.param("id"));
let videoId: string | number = id as string;
@ -50,27 +45,33 @@ export const videoInfoHandler = createHandlers(async (c: ContextType) => {
}
const cacheKey = `cvsa:videoInfo:${videoId}`;
endTime(c, 'parse');
startTime(c, 'cache', 'Check for cached data');
const cachedData = await redis.get(cacheKey);
endTime(c, 'cache');
if (cachedData) {
return c.json(JSON.parse(cachedData));
}
startTime(c, 'net', 'Fetch data');
let result: VideoInfoData | number;
if (typeof videoId === "number") {
result = await getVideoInfo(videoId, "getVideoInfo");
} else {
result = await getVideoInfoByBV(videoId, "getVideoInfo");
}
endTime(c, 'net');
if (typeof result === "number") {
return c.json({ message: "Error fetching video info", code: result }, 500);
}
startTime(c, 'db', 'Write data to database');
await redis.setex(cacheKey, CACHE_EXPIRATION_SECONDS, JSON.stringify(result));
await insertVideoSnapshot(client, result);
await insertVideoSnapshot(result);
endTime(c, 'db');
return c.json(result);
} catch (e) {
if (e instanceof ValidationError) {

View File

@ -0,0 +1,58 @@
import { sql } from "./db";
import type { VideoSnapshotType } from "@core/db/schema.d.ts";
export async function getVideoSnapshots(
aid: number,
limit: number,
pageOrOffset: number,
reverse: boolean,
mode: "page" | "offset" = "page"
) {
const offset = mode === "page" ? (pageOrOffset - 1) * limit : pageOrOffset;
if (reverse) {
return sql<VideoSnapshotType[]>`
SELECT *
FROM video_snapshot
WHERE aid = ${aid}
ORDER BY created_at
LIMIT ${limit} OFFSET ${offset}
`;
} else {
return sql<VideoSnapshotType[]>`
SELECT *
FROM video_snapshot
WHERE aid = ${aid}
ORDER BY created_at DESC
LIMIT ${limit} OFFSET ${offset}
`;
}
}
export async function getVideoSnapshotsByBV(
bv: string,
limit: number,
pageOrOffset: number,
reverse: boolean,
mode: "page" | "offset" = "page"
) {
const offset = mode === "page" ? (pageOrOffset - 1) * limit : pageOrOffset;
if (reverse) {
return sql<VideoSnapshotType[]>`
SELECT vs.*
FROM video_snapshot vs
JOIN bilibili_metadata bm ON vs.aid = bm.aid
WHERE bm.bvid = ${bv}
ORDER BY vs.created_at
LIMIT ${limit} OFFSET ${offset}
`;
} else {
return sql<VideoSnapshotType[]>`
SELECT vs.*
FROM video_snapshot vs
JOIN bilibili_metadata bm ON vs.aid = bm.aid
WHERE bm.bvid = ${bv}
ORDER BY vs.created_at DESC
LIMIT ${limit} OFFSET ${offset}
`;
}
}

View File

@ -1,22 +0,0 @@
{
"name": "@cvsa/backend",
"imports": {
"@rabbit-company/argon2id": "jsr:@rabbit-company/argon2id@^2.1.0",
"hono": "jsr:@hono/hono@^4.7.5",
"zod": "npm:zod",
"yup": "npm:yup",
"@core/": "../core/",
"log/": "../core/log/",
"@crawler/net/videoInfo": "../crawler/net/getVideoInfo.ts",
"ioredis": "npm:ioredis"
},
"tasks": {
"dev": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --watch main.ts",
"start": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --host 127.0.0.1 main.ts"
},
"compilerOptions": {
"jsx": "precompile",
"jsxImportSource": "hono/jsx"
},
"exports": "./main.ts"
}

View File

@ -1,26 +0,0 @@
import { Hono } from "hono";
import { dbCredMiddleware, dbMiddleware } from "./database.ts";
import { rootHandler } from "./root.ts";
import { getSnapshotsHanlder } from "./snapshots.ts";
import { registerHandler } from "./register.ts";
import { videoInfoHandler } from "./videoInfo.ts";
export const app = new Hono();
app.use("/video/*", dbMiddleware);
app.use("/user", dbCredMiddleware);
app.get("/", ...rootHandler);
app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
app.post("/user", ...registerHandler);
app.get("/video/:id/info", ...videoInfoHandler);
const fetch = app.fetch;
export default {
fetch,
} satisfies Deno.ServeDefaultExport;
export const VERSION = "0.4.2";

View File

@ -0,0 +1,14 @@
import { bodyLimit } from "hono/body-limit";
import { ErrorResponse } from "../src/schema";
export const bodyLimitForPing = bodyLimit({
maxSize: 14000,
onError: (c) => {
const res: ErrorResponse<string> = {
message: "Body too large",
errors: ["Body should not be larger than 14kB."],
code: "BODY_TOO_LARGE"
};
return c.json(res, 413);
}
});

View File

@ -0,0 +1,6 @@
import { Context, Next } from "hono";
export const contentType = async (c: Context, next: Next) => {
await next();
c.header("Content-Type", "application/json; charset=utf-8");
};

View File

@ -0,0 +1,19 @@
import { Hono } from "hono";
import { Variables } from "hono/types";
import { bodyLimitForPing } from "./bodyLimits.ts";
import { pingHandler } from "../routes/ping.ts";
import { registerRateLimiter } from "./rateLimiters.ts";
import { preetifyResponse } from "./preetifyResponse.ts";
import { logger } from "./logger.ts";
import { timing } from "hono/timing";
import { contentType } from "./contentType.ts";
export function configureMiddleWares(app: Hono<{Variables: Variables }>) {
app.use("*", contentType);
app.use(timing());
app.use("*", preetifyResponse);
app.use("*", logger({}));
app.post("/user", registerRateLimiter);
app.all("/ping", bodyLimitForPing, ...pingHandler);
}

View File

@ -0,0 +1,160 @@
// Color constants
import { Context, Next } from "hono";
import { TimingVariables } from "hono/timing";
import { getConnInfo } from "hono/bun";
const green = "\x1b[97;42m";
const white = "\x1b[90;47m";
const yellow = "\x1b[90;43m";
const red = "\x1b[97;41m";
const blue = "\x1b[97;44m";
const magenta = "\x1b[97;45m";
const cyan = "\x1b[97;46m";
const reset = "\x1b[0m";
let consoleColorMode = "auto";
function formatCurrentTime() {
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, "0"); // Month is 0-indexed
const day = String(now.getDate()).padStart(2, "0");
const hours = String(now.getHours()).padStart(2, "0");
const minutes = String(now.getMinutes()).padStart(2, "0");
const seconds = String(now.getSeconds()).padStart(2, "0");
const milliseconds = String(now.getMilliseconds()).padStart(3, "0");
return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}.${milliseconds}`;
}
export const DisableConsoleColor = () => {
consoleColorMode = "disable";
};
export const ForceConsoleColor = () => {
consoleColorMode = "force";
};
const defaultFormatter = (params) => {
const latency = params.latency > 60000 ? `${Math.round(params.latency / 1000)}s` : `${params.latency}ms`;
let statusColor = white;
if (params.isOutputColor) {
if (params.status >= 100 && params.status < 300) statusColor = green;
else if (params.status >= 300 && params.status < 400) statusColor = white;
else if (params.status >= 400 && params.status < 500) statusColor = yellow;
else statusColor = red;
}
let methodColor = reset;
switch (params.method) {
case "GET":
methodColor = blue;
break;
case "POST":
methodColor = cyan;
break;
case "PUT":
methodColor = yellow;
break;
case "DELETE":
methodColor = red;
break;
case "PATCH":
methodColor = green;
break;
case "HEAD":
methodColor = magenta;
break;
case "OPTIONS":
methodColor = white;
break;
}
return (
`${params.timestamp} |${statusColor} ${params.status} ${reset}| ` +
`${latency.padStart(7)} | ${params.ip.padStart(16)} |` +
`${methodColor} ${params.method.padEnd(6)}${reset} ${params.path}`
);
};
type Ctx = Context
export const logger = (config) => {
const { formatter = defaultFormatter, output = console, skipPaths = [], skip = null } = config;
// Convert skipPaths to Set for faster lookups
const skipPathsSet = new Set(skipPaths);
return async (c: Ctx, next: Next) => {
const start = Date.now();
const url = new URL(c.req.url);
const path = url.pathname;
// Check if we should skip logging
if (skipPathsSet.has(path) || (typeof skip === "function" && skip(c))) {
return next();
}
try {
await next();
} catch (error) {
// Handle errors
const errorParams = {
timestamp: formatCurrentTime(),
latency: Date.now() - start,
status: 500,
ip: getClientIP(c),
method: c.req.method,
path,
error: error.message,
isOutputColor: shouldColorize(c)
};
output.error(
formatter({
...errorParams,
errorMessage: error.message
})
);
throw error;
}
const status = c.res.status;
const latency = Date.now() - start;
const params = {
timestamp: formatCurrentTime(),
latency,
status,
ip: getClientIP(c),
method: c.req.method,
path,
bodySize: c.res.headers.get("content-length") || 0,
isOutputColor: shouldColorize(c)
};
// Format and output the log
const logMessage = formatter(params);
if (status >= 400 && status < 500) {
output.warn?.(logMessage) || output.log(logMessage);
} else if (status >= 500) {
output.error?.(logMessage) || output.log(logMessage);
} else {
output.log(logMessage);
}
};
};
function shouldColorize(c) {
if (consoleColorMode === "disable") return false;
if (consoleColorMode === "force") return true;
// In development environment with TTY
return process.stdout.isTTY;
}
export function getClientIP(c: Ctx) {
const info = getConnInfo(c);
return info.remote.address;
}

View File

@ -0,0 +1,18 @@
import { startTime, endTime } from "hono/timing";
import { Context, Next } from "hono";
export const preetifyResponse = async (c: Context, next: Next) => {
await next();
const contentType = c.res.headers.get("Content-Type") || "";
if (!contentType.includes("application/json")) return;
const accept = c.req.header("Accept") || "";
const secFetchMode = c.req.header("Sec-Fetch-Mode");
const isBrowser = accept.includes("text/html") || secFetchMode === "navigate";
if (isBrowser) {
const json = await c.res.json();
startTime(c, "seralize", "Prettify the response");
const prettyJson = JSON.stringify(json, null, 2);
endTime(c, "seralize");
c.res = new Response(prettyJson, { headers: { "Content-Type": "text/plain; charset=utf-8" } });
}
};

View File

@ -0,0 +1,27 @@
import { rateLimiter, Store } from "hono-rate-limiter";
import type { BlankEnv } from "hono/types";
import { MINUTE } from "@core/const/time.ts";
import { getConnInfo } from "hono/bun";
import type { Context } from "hono";
import { redis } from "@core/db/redis.ts";
import { RedisStore } from "rate-limit-redis";
export const registerRateLimiter = rateLimiter<BlankEnv, "/user", {}>({
windowMs: 60 * MINUTE,
limit: 10,
standardHeaders: "draft-6",
keyGenerator: (c) => {
const info = getConnInfo(c as unknown as Context<BlankEnv, "/user", {}>);
if (!info.remote || !info.remote.address) {
return crypto.randomUUID();
}
const addr = info.remote.address;
const path = new URL(c.req.url).pathname;
const method = c.req.method;
return `${method}-${path}@${addr}`;
},
store: new RedisStore({
// @ts-expect-error - Known issue: the `c`all` function is not present in @types/ioredis
sendCommand: (...args: string[]) => redis.call(...args)
}) as unknown as Store
});

View File

@ -0,0 +1,22 @@
{
"name": "backend",
"scripts": {
"format": "prettier --write .",
"dev": "NODE_ENV=development bun run --hot src/main.ts",
"start": "NODE_ENV=production bun run src/main.ts"
},
"dependencies": {
"@rabbit-company/argon2id": "^2.1.0",
"hono": "^4.7.8",
"hono-rate-limiter": "^0.4.2",
"ioredis": "^5.6.1",
"postgres": "^3.4.5",
"rate-limit-redis": "^4.2.0",
"yup": "^1.6.1",
"zod": "^3.24.3"
},
"devDependencies": {
"@types/bun": "^1.2.11",
"prettier": "^3.5.3"
}
}

View File

@ -1,65 +0,0 @@
import { createHandlers } from "./utils.ts";
import Argon2id from "@rabbit-company/argon2id";
import { object, string, ValidationError } from "yup";
import type { Context } from "hono";
import type { Bindings, BlankEnv, BlankInput } from "hono/types";
import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
const RegistrationBodySchema = object({
username: string().trim().required("Username is required").max(50, "Username cannot exceed 50 characters"),
password: string().required("Password is required"),
nickname: string().optional(),
});
type ContextType = Context<BlankEnv & { Bindings: Bindings }, "/user", BlankInput>;
export const userExists = async (username: string, client: Client) => {
const query = `
SELECT * FROM users WHERE username = $1
`;
const result = await client.queryObject(query, [username]);
return result.rows.length > 0;
};
export const registerHandler = createHandlers(async (c: ContextType) => {
const client = c.get("dbCred");
try {
const body = await RegistrationBodySchema.validate(await c.req.json());
const { username, password, nickname } = body;
if (await userExists(username, client)) {
return c.json({
message: `User "${username}" already exists.`,
}, 400);
}
const hash = await Argon2id.hashEncoded(password);
const query = `
INSERT INTO users (username, password, nickname) VALUES ($1, $2, $3)
`;
await client.queryObject(query, [username, hash, nickname || null]);
return c.json({
message: `User "${username}" registered successfully.`,
}, 201);
} catch (e) {
if (e instanceof ValidationError) {
return c.json({
message: "Invalid registration data.",
errors: e.errors,
}, 400);
} else if (e instanceof SyntaxError) {
return c.json({
message: "Invalid JSON in request body.",
}, 400);
} else {
console.error("Registration error:", e);
return c.json({
message: "An unexpected error occurred during registration.",
error: (e as Error).message,
}, 500);
}
}
});

View File

@ -0,0 +1,10 @@
import { Context } from "hono";
export const notFoundRoute = (c: Context) => {
return c.json(
{
message: "Not Found"
},
404
);
};

View File

@ -0,0 +1,17 @@
import { rootHandler } from "./root/root.ts";
import { pingHandler } from "./ping.ts";
import { getSnapshotsHanlder } from "./snapshots.ts";
import { registerHandler } from "./user.ts";
import { videoInfoHandler } from "db/videoInfo.ts";
import { Hono } from "hono";
import { Variables } from "hono/types";
export function configureRoutes(app: Hono<{Variables: Variables }>) {
app.get("/", ...rootHandler);
app.all("/ping", ...pingHandler);
app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
app.post("/user", ...registerHandler);
app.get("/video/:id/info", ...videoInfoHandler);
}

View File

@ -0,0 +1,24 @@
import { getClientIP } from "middleware/logger.ts";
import { createHandlers } from "../src/utils.ts";
import { VERSION } from "../src/main.ts";
export const pingHandler = createHandlers(async (c) => {
const requestHeaders = c.req.raw.headers;
return c.json({
"message": "pong",
"request": {
"headers": requestHeaders,
"ip": getClientIP(c),
"mode": c.req.raw.mode,
"method": c.req.method,
"query": new URL(c.req.url).searchParams,
"body": await c.req.text(),
"url": c.req.raw.url
},
"response": {
"time": new Date().getTime(),
"status": 200,
"version": VERSION,
}
});
});

View File

@ -1,6 +1,6 @@
import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "./singers.ts";
import { VERSION } from "./main.ts";
import { createHandlers } from "./utils.ts";
import { VERSION } from "../../src/main.ts";
import { createHandlers } from "../../src/utils.ts";
export const rootHandler = createHandlers((c) => {
let singer: Singer | Singer[];
@ -17,13 +17,13 @@ export const rootHandler = createHandlers((c) => {
singer = pickSinger();
}
return c.json({
"project": {
"name": "中V档案馆",
"motto": "一起唱吧,心中的歌!",
project: {
name: "中V档案馆",
motto: "一起唱吧,心中的歌!"
},
"status": 200,
"version": VERSION,
"time": Date.now(),
"singer": singer,
status: 200,
version: VERSION,
time: Date.now(),
singer: singer
});
});

View File

@ -0,0 +1,103 @@
export const singers = [
{
name: "洛天依",
color: "#66CCFF",
birthday: "0712"
},
{
name: "言和",
color: "#00FFCC",
birthday: "0711"
},
{
name: "乐正绫",
color: "#EE0000",
birthday: "0412"
},
{
name: "乐正龙牙",
color: "#006666",
birthday: "1002"
},
{
name: "徵羽摩柯",
color: "#0080FF",
birthday: "1210"
},
{
name: "墨清弦",
color: "#FFFF00",
birthday: "0520"
},
{
name: "星尘",
color: "#9999FF",
birthday: "0812"
},
{
name: "心华",
color: "#EE82EE",
birthday: "0210"
},
{
name: "海伊",
color: "#3399FF",
birthday: "0722"
},
{
name: "苍穹",
color: "#8BC0B5",
birthday: "0520"
},
{
name: "赤羽",
color: "#FF4004",
birthday: "1126"
},
{
name: "诗岸",
color: "#F6BE72",
birthday: "0119"
},
{
name: "牧心",
color: "#2A2859",
birthday: "0807"
}
];
export interface Singer {
name: string;
color?: string;
birthday?: string;
message?: string;
}
export const specialSingers = [
{
name: "雅音宫羽",
message: "你是我最真模样,从来不曾遗忘。"
},
{
name: "初音未来",
message: "初始之音,响彻未来!"
}
];
export const pickSinger = () => {
const index = Math.floor(Math.random() * singers.length);
return singers[index];
};
export const pickSpecialSinger = () => {
const index = Math.floor(Math.random() * specialSingers.length);
return specialSingers[index];
};
export const getSingerForBirthday = (): Singer[] => {
const today = new Date();
const month = String(today.getMonth() + 1).padStart(2, "0");
const day = String(today.getDate()).padStart(2, "0");
const datestring = `${month}${day}`;
return singers.filter((singer) => singer.birthday === datestring);
};

View File

@ -1,22 +1,24 @@
import type { Context } from "hono";
import { createHandlers } from "./utils.ts";
import { createHandlers } from "../src/utils.ts";
import type { BlankEnv, BlankInput } from "hono/types";
import { getVideoSnapshots, getVideoSnapshotsByBV } from "@core/db/videoSnapshot.ts";
import { getVideoSnapshots, getVideoSnapshotsByBV } from "../db/videoSnapshot.ts";
import type { VideoSnapshotType } from "@core/db/schema.d.ts";
import { boolean, mixed, number, object, ValidationError } from "yup";
import { ErrorResponse } from "../src/schema";
import { startTime, endTime } from "hono/timing";
const SnapshotQueryParamsSchema = object({
ps: number().integer().optional().positive(),
pn: number().integer().optional().positive(),
offset: number().integer().optional().positive(),
reverse: boolean().optional(),
reverse: boolean().optional()
});
export const idSchema = mixed().test(
"is-valid-id",
'id must be a string starting with "av" followed by digits, or "BV" followed by 10 alphanumeric characters, or a positive integer',
async (value) => {
if (value && await number().integer().isValid(value)) {
if (value && (await number().integer().isValid(value))) {
const v = parseInt(value as string);
return Number.isInteger(v) && v > 0;
}
@ -34,13 +36,12 @@ export const idSchema = mixed().test(
}
return false;
},
}
);
type ContextType = Context<BlankEnv, "/video/:id/snapshots", BlankInput>;
export const getSnapshotsHanlder = createHandlers(async (c: ContextType) => {
const client = c.get("db");
startTime(c, "parse", "Parse the request");
try {
const idParam = await idSchema.validate(c.req.param("id"));
let videoId: string | number = idParam as string;
@ -70,23 +71,36 @@ export const getSnapshotsHanlder = createHandlers(async (c: ContextType) => {
let result: VideoSnapshotType[];
endTime(c, "parse");
startTime(c, "db", "Query the database");
if (typeof videoId === "number") {
result = await getVideoSnapshots(client, videoId, limit, pageOrOffset, reverse, mode);
result = await getVideoSnapshots(videoId, limit, pageOrOffset, reverse, mode);
} else {
result = await getVideoSnapshotsByBV(client, videoId, limit, pageOrOffset, reverse, mode);
result = await getVideoSnapshotsByBV(videoId, limit, pageOrOffset, reverse, mode);
}
endTime(c, "db");
const rows = result.map((row) => ({
...row,
aid: Number(row.aid),
aid: Number(row.aid)
}));
return c.json(rows);
} catch (e) {
} catch (e: unknown) {
if (e instanceof ValidationError) {
return c.json({ message: "Invalid query parameters", errors: e.errors }, 400);
const response: ErrorResponse<string> = {
code: "INVALID_QUERY_PARAMS",
message: "Invalid query parameters",
errors: e.errors
};
return c.json<ErrorResponse<string>>(response, 400);
} else {
return c.json({ message: "Unhandled error", error: e }, 500);
const response: ErrorResponse<unknown> = {
code: "UNKNOWN_ERR",
message: "Unhandled error",
errors: [e]
};
return c.json<ErrorResponse<unknown>>(response, 500);
}
}
});

View File

@ -0,0 +1,74 @@
import { createHandlers } from "src/utils.ts";
import Argon2id from "@rabbit-company/argon2id";
import { object, string, ValidationError } from "yup";
import type { Context } from "hono";
import type { Bindings, BlankEnv, BlankInput } from "hono/types";
import { sqlCred } from "db/db.ts";
import { ErrorResponse, StatusResponse } from "src/schema";
const RegistrationBodySchema = object({
username: string().trim().required("Username is required").max(50, "Username cannot exceed 50 characters"),
password: string().required("Password is required"),
nickname: string().optional()
});
type ContextType = Context<BlankEnv & { Bindings: Bindings }, "/user", BlankInput>;
export const userExists = async (username: string) => {
const result = await sqlCred`
SELECT 1
FROM users
WHERE username = ${username}
`;
return result.length > 0;
};
export const registerHandler = createHandlers(async (c: ContextType) => {
try {
const body = await RegistrationBodySchema.validate(await c.req.json());
const { username, password, nickname } = body;
if (await userExists(username)) {
const response: StatusResponse = {
message: `User "${username}" already exists.`
};
return c.json<StatusResponse>(response, 400);
}
const hash = await Argon2id.hashEncoded(password);
await sqlCred`
INSERT INTO users (username, password, nickname)
VALUES (${username}, ${hash}, ${nickname ? nickname : null})
`;
const response: StatusResponse = {
message: `User '${username}' registered successfully.`
}
return c.json<StatusResponse>(response, 201);
} catch (e) {
if (e instanceof ValidationError) {
const response: ErrorResponse<string> = {
message: "Invalid registration data.",
errors: e.errors,
code: "INVALID_PAYLOAD"
}
return c.json<ErrorResponse<string>>(response, 400);
} else if (e instanceof SyntaxError) {
const response: ErrorResponse<string> = {
message: "Invalid JSON payload.",
errors: [e.message],
code: "INVALID_FORMAT"
}
return c.json<ErrorResponse<string>>(response, 400);
} else {
const response: ErrorResponse<string> = {
message: "Invalid JSON payload.",
errors: [(e as Error).message],
code: "UNKNOWN_ERR"
}
return c.json<ErrorResponse<string>>(response, 500);
}
}
});

View File

@ -1,103 +0,0 @@
export const singers = [
{
"name": "洛天依",
"color": "#66CCFF",
"birthday": "0712",
},
{
"name": "言和",
"color": "#00FFCC",
"birthday": "0711",
},
{
"name": "乐正绫",
"color": "#EE0000",
"birthday": "0412",
},
{
"name": "乐正龙牙",
"color": "#006666",
"birthday": "1002",
},
{
"name": "徵羽摩柯",
"color": "#0080FF",
"birthday": "1210",
},
{
"name": "墨清弦",
"color": "#FFFF00",
"birthday": "0520",
},
{
"name": "星尘",
"color": "#9999FF",
"birthday": "0812",
},
{
"name": "心华",
"color": "#EE82EE",
"birthday": "0210",
},
{
"name": "海伊",
"color": "#3399FF",
"birthday": "0722",
},
{
"name": "苍穹",
"color": "#8BC0B5",
"birthday": "0520",
},
{
"name": "赤羽",
"color": "#FF4004",
"birthday": "1126",
},
{
"name": "诗岸",
"color": "#F6BE72",
"birthday": "0119",
},
{
"name": "牧心",
"color": "#2A2859",
"birthday": "0807",
},
];
export interface Singer {
name: string;
color?: string;
birthday?: string;
message?: string;
}
export const specialSingers = [
{
"name": "雅音宫羽",
"message": "你是我最真模样,从来不曾遗忘。",
},
{
"name": "初音未来",
"message": "初始之音,响彻未来!",
},
];
export const pickSinger = () => {
const index = Math.floor(Math.random() * singers.length);
return singers[index];
};
export const pickSpecialSinger = () => {
const index = Math.floor(Math.random() * specialSingers.length);
return specialSingers[index];
};
export const getSingerForBirthday = (): Singer[] => {
const today = new Date();
const month = String(today.getMonth() + 1).padStart(2, "0");
const day = String(today.getDate()).padStart(2, "0");
const datestring = `${month}${day}`;
return singers.filter((singer) => singer.birthday === datestring);
};

View File

@ -0,0 +1,18 @@
import { Hono } from "hono";
import type { TimingVariables } from "hono/timing";
import { startServer } from "./startServer.ts";
import { configureRoutes } from "routes";
import { configureMiddleWares } from "middleware";
import { notFoundRoute } from "routes/404.ts";
type Variables = TimingVariables;
const app = new Hono<{ Variables: Variables }>();
app.notFound(notFoundRoute);
configureMiddleWares(app);
configureRoutes(app);
await startServer(app);
export const VERSION = "0.4.6";

11
packages/backend/src/schema.d.ts vendored Normal file
View File

@ -0,0 +1,11 @@
type ErrorCode = "INVALID_QUERY_PARAMS" | "UNKNOWN_ERR" | "INVALID_PAYLOAD" | "INVALID_FORMAT" | "BODY_TOO_LARGE";
export interface ErrorResponse<E> {
code: ErrorCode
message: string;
errors: E[];
}
export interface StatusResponse {
message: string;
}

View File

@ -0,0 +1,91 @@
import { serve } from "bun";
import { Hono } from "hono";
import os from "os";
import { BlankSchema, Variables } from "hono/types";
function getLocalIpAddress(): string {
const interfaces = os.networkInterfaces();
for (const name of Object.keys(interfaces)) {
for (const iface of interfaces[name]!) {
if (iface.family === "IPv4" && !iface.internal) {
return iface.address;
}
}
}
return "localhost";
}
function logStartup(hostname: string, port: number, wasAutoIncremented: boolean, originalPort: number) {
const localUrl = `http://localhost:${port}`;
const networkIp = hostname === "0.0.0.0" ? getLocalIpAddress() : "";
const networkUrl = networkIp ? `http://${networkIp}:${port}` : "";
console.log("\n");
console.log("🚀 Server is running at:");
console.log(`> Local: ${localUrl}`);
if (networkIp) {
console.log(`> Network: ${networkUrl}`);
}
if (wasAutoIncremented) {
console.log(`\n⚠ Port ${originalPort} is in use, using port ${port} instead.`);
}
console.log("\nPress Ctrl+C to quit.");
}
export async function startServer(app: Hono<{Variables: Variables }>) {
const NODE_ENV = process.env.NODE_ENV || "production";
const HOST = process.env.HOST ?? (NODE_ENV === "development" ? "0.0.0.0" : "127.0.0.1");
const PORT = process.env.PORT ? parseInt(process.env.PORT, 10) : undefined;
const DEFAULT_PORT = 3000;
const MAX_ATTEMPTS = 15;
if (PORT !== undefined) {
try {
const server = serve({
fetch: app.fetch,
hostname: HOST,
port: PORT
});
logStartup(HOST, PORT, false, DEFAULT_PORT);
return server;
} catch (e: any) {
console.error(`Failed to start server on port ${PORT}:`, e.message);
process.exit(1);
}
}
let attemptPort = DEFAULT_PORT;
let success = false;
let error: unknown = null;
for (let i = 0; i <= MAX_ATTEMPTS; i++) {
try {
const server = serve({
fetch: app.fetch,
hostname: HOST,
port: attemptPort
});
const wasAutoIncremented = attemptPort !== DEFAULT_PORT;
logStartup(HOST, attemptPort, wasAutoIncremented, DEFAULT_PORT);
return server;
} catch (e: any) {
if (e.code === "EADDRINUSE") {
attemptPort++;
} else {
error = e;
break;
}
}
}
if (!success) {
console.error(`Could not find an available port after ${MAX_ATTEMPTS + 1} attempts.`);
if (error) {
console.error("Last error:", (error as Error).message);
}
process.exit(1);
}
}

View File

@ -0,0 +1,19 @@
{
"include": ["**/*.ts"],
"compilerOptions": {
"baseUrl": ".",
"target": "ESNext",
"module": "ESNext",
"useDefineForClassFields": true,
"moduleResolution": "bundler",
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"skipLibCheck": true,
"paths": {
"@core/*": ["../core/*"],
"@crawler/*": ["../crawler/*"]
},
"allowSyntheticDefaultImports": true,
"allowImportingTsExtensions": true,
"noEmit": true
}
}

104
packages/core/bun.lock Normal file
View File

@ -0,0 +1,104 @@
{
"lockfileVersion": 1,
"workspaces": {
"": {
"dependencies": {
"chalk": "^5.4.1",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"postgres": "^3.4.5",
"winston": "^3.17.0",
},
"devDependencies": {
"@types/ioredis": "^5.0.0",
},
},
},
"packages": {
"@colors/colors": ["@colors/colors@1.6.0", "", {}, "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA=="],
"@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="],
"@ioredis/commands": ["@ioredis/commands@1.2.0", "", {}, "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="],
"@types/ioredis": ["@types/ioredis@5.0.0", "", { "dependencies": { "ioredis": "*" } }, "sha512-zJbJ3FVE17CNl5KXzdeSPtdltc4tMT3TzC6fxQS0sQngkbFZ6h+0uTafsRqu+eSLIugf6Yb0Ea0SUuRr42Nk9g=="],
"@types/triple-beam": ["@types/triple-beam@1.3.5", "", {}, "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw=="],
"async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="],
"chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="],
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
"color": ["color@3.2.1", "", { "dependencies": { "color-convert": "^1.9.3", "color-string": "^1.6.0" } }, "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA=="],
"color-convert": ["color-convert@1.9.3", "", { "dependencies": { "color-name": "1.1.3" } }, "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg=="],
"color-name": ["color-name@1.1.3", "", {}, "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="],
"color-string": ["color-string@1.9.1", "", { "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" } }, "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg=="],
"colorspace": ["colorspace@1.1.4", "", { "dependencies": { "color": "^3.1.3", "text-hex": "1.0.x" } }, "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w=="],
"debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
"denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="],
"enabled": ["enabled@2.0.0", "", {}, "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ=="],
"fecha": ["fecha@4.2.3", "", {}, "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw=="],
"fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="],
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
"ioredis": ["ioredis@5.6.1", "", { "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA=="],
"is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="],
"is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="],
"kuler": ["kuler@2.0.0", "", {}, "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="],
"lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="],
"lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="],
"logform": ["logform@2.7.0", "", { "dependencies": { "@colors/colors": "1.6.0", "@types/triple-beam": "^1.3.2", "fecha": "^4.2.0", "ms": "^2.1.1", "safe-stable-stringify": "^2.3.1", "triple-beam": "^1.3.0" } }, "sha512-TFYA4jnP7PVbmlBIfhlSe+WKxs9dklXMTEGcBCIvLhE/Tn3H6Gk1norupVW7m5Cnd4bLcr08AytbyV/xj7f/kQ=="],
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
"one-time": ["one-time@1.0.0", "", { "dependencies": { "fn.name": "1.x.x" } }, "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g=="],
"postgres": ["postgres@3.4.5", "", {}, "sha512-cDWgoah1Gez9rN3H4165peY9qfpEo+SA61oQv65O3cRUE1pOEoJWwddwcqKE8XZYjbblOJlYDlLV4h67HrEVDg=="],
"readable-stream": ["readable-stream@3.6.2", "", { "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", "util-deprecate": "^1.0.1" } }, "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA=="],
"redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="],
"redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="],
"safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="],
"safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="],
"simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="],
"stack-trace": ["stack-trace@0.0.10", "", {}, "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg=="],
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
"string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="],
"text-hex": ["text-hex@1.0.0", "", {}, "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="],
"triple-beam": ["triple-beam@1.4.1", "", {}, "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg=="],
"util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="],
"winston": ["winston@3.17.0", "", { "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", "logform": "^2.7.0", "one-time": "^1.0.0", "readable-stream": "^3.4.0", "safe-stable-stringify": "^2.3.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", "winston-transport": "^4.9.0" } }, "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw=="],
"winston-transport": ["winston-transport@4.9.0", "", { "dependencies": { "logform": "^2.7.0", "readable-stream": "^3.6.2", "triple-beam": "^1.3.0" } }, "sha512-8drMJ4rkgaPo1Me4zD/3WLfI/zPdA9o2IipKODunnGDcuqbHwjsbB79ylv04LCGGzU0xQ6vTznOMpQGaLhhm6A=="],
}
}

View File

@ -0,0 +1,5 @@
export const SECOND = 1000;
export const MINUTE = 60 * SECOND;
export const HOUR = 60 * MINUTE;
export const DAY = 24 * HOUR;
export const WEEK = 7 * DAY;

View File

@ -0,0 +1,4 @@
import postgres from "postgres";
import { postgresConfigNpm } from "./pgConfigNew";
export const sql = postgres(postgresConfigNpm);

View File

@ -21,6 +21,14 @@ export const postgresConfig = {
password: databasePassword,
};
export const postgresConfigNpm = {
host: databaseHost,
port: parseInt(databasePort),
database: databaseName,
username: databaseUser,
password: databasePassword,
}
export const postgresConfigCred = {
hostname: databaseHost,
port: parseInt(databasePort),

View File

@ -0,0 +1,38 @@
const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT", "DB_NAME_CRED"];
const unsetVars = requiredEnvVars.filter((key) => process.env[key] === undefined);
if (unsetVars.length > 0) {
throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`);
}
const databaseHost = process.env["DB_HOST"]!;
const databaseName = process.env["DB_NAME"];
const databaseNameCred = process.env["DB_NAME_CRED"]!;
const databaseUser = process.env["DB_USER"]!;
const databasePassword = process.env["DB_PASSWORD"]!;
const databasePort = process.env["DB_PORT"]!;
export const postgresConfig = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseName,
user: databaseUser,
password: databasePassword
};
export const postgresConfigNpm = {
host: databaseHost,
port: parseInt(databasePort),
database: databaseName,
username: databaseUser,
password: databasePassword
};
export const postgresConfigCred = {
hostname: databaseHost,
port: parseInt(databasePort),
database: databaseNameCred,
user: databaseUser,
password: databasePassword
};

View File

@ -1,3 +1,10 @@
import { Redis } from "ioredis";
export const redis = new Redis({ maxRetriesPerRequest: null });
const host = process.env.REDIS_HOST || "localhost";
const port = parseInt(process.env.REDIS_PORT) || 6379;
export const redis = new Redis({
port: port,
host: host,
maxRetriesPerRequest: null,
});

View File

@ -1,12 +1,17 @@
{
"name": "@cvsa/core",
"exports": "./main.ts",
"imports": {
"ioredis": "npm:ioredis",
"log/": "./log/",
"db/": "./db/",
"$std/": "https://deno.land/std@0.216.0/",
"mq/": "./mq/",
"chalk": "npm:chalk"
}
"name": "@cvsa/core",
"exports": "./main.ts",
"imports": {
"ioredis": "npm:ioredis",
"log/": "./log/",
"db/": "./db/",
"$std/": "https://deno.land/std@0.216.0/",
"mq/": "./mq/",
"chalk": "npm:chalk",
"winston": "npm:winston",
"logform": "npm:logform",
"@core/": "./",
"child_process": "node:child_process",
"util": "node:util"
}
}

View File

@ -1,5 +1,5 @@
import winston, { format, transports } from "npm:winston";
import type { TransformableInfo } from "npm:logform";
import winston, { format, transports } from "winston";
import type { TransformableInfo } from "logform";
import chalk from "chalk";
const customFormat = format.printf((info: TransformableInfo) => {
@ -25,12 +25,12 @@ const createTransport = (level: string, filename: string) => {
let maxFiles = undefined;
let tailable = undefined;
if (level === "verbose") {
maxsize = 10 * MB;
maxsize = 500 * MB;
maxFiles = 10;
tailable = false;
} else if (level === "warn") {
maxsize = 10 * MB;
maxFiles = 1;
maxFiles = 5;
tailable = false;
}
function replacer(key: unknown, value: unknown) {
@ -52,9 +52,9 @@ const createTransport = (level: string, filename: string) => {
});
};
const sillyLogPath = Deno.env.get("LOG_VERBOSE") ?? "logs/verbose.log";
const warnLogPath = Deno.env.get("LOG_WARN") ?? "logs/warn.log";
const errorLogPath = Deno.env.get("LOG_ERROR") ?? "logs/error.log";
const sillyLogPath = process.env["LOG_VERBOSE"] ?? "logs/verbose.log";
const warnLogPath = process.env["LOG_WARN"] ?? "logs/warn.log";
const errorLogPath = process.env["LOG_ERROR"] ?? "logs/error.log";
const winstonLogger = winston.createLogger({
levels: winston.config.npm.levels,
@ -62,7 +62,7 @@ const winstonLogger = winston.createLogger({
new transports.Console({
level: "debug",
format: format.combine(
format.timestamp({ format: "HH:mm:ss.SSS" }),
format.timestamp({ format: "YYYY-MM-DD HH:mm:ss.SSS" }),
format.colorize(),
format.errors({ stack: true }),
customFormat,

View File

@ -1,4 +1,4 @@
import { SlidingWindow } from "./slidingWindow.ts";
import type { SlidingWindow } from "./slidingWindow.ts";
export interface RateLimiterConfig {
window: SlidingWindow;

View File

@ -1,4 +1,4 @@
import { Redis } from "ioredis";
import type { Redis } from "ioredis";
export class SlidingWindow {
private redis: Redis;

View File

@ -2,8 +2,42 @@ import logger from "log/logger.ts";
import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts";
import { SlidingWindow } from "mq/slidingWindow.ts";
import { redis } from "db/redis.ts";
import Redis from "ioredis";
import { SECOND } from "$std/datetime/constants.ts";
import { ReplyError } from "ioredis";
import { SECOND } from "../const/time.ts";
import { spawn, SpawnOptions } from "child_process";
export function spawnPromise(
command: string,
args: string[] = [],
options?: SpawnOptions,
): Promise<{ stdout: string; stderr: string }> {
return new Promise((resolve, reject) => {
const child = spawn(command, args, options);
let stdout = "";
let stderr = "";
child.stdout?.on("data", (data) => {
stdout += data;
});
child.stderr?.on("data", (data) => {
stderr += data;
});
child.on("close", (code) => {
if (code !== 0) {
reject(new Error(`Error code: ${code}\nstderr: ${stderr}`));
} else {
resolve({ stdout, stderr });
}
});
child.on("error", (err) => {
reject(err);
});
});
}
interface Proxy {
type: string;
@ -99,7 +133,7 @@ class NetworkDelegate {
await this.providerLimiters[providerLimiterId]?.trigger();
} catch (e) {
const error = e as Error;
if (e instanceof Redis.ReplyError) {
if (e instanceof ReplyError) {
logger.error(error, "redis");
}
logger.warn(`Unhandled error: ${error.message}`, "mq", "proxyRequest");
@ -209,7 +243,7 @@ class NetworkDelegate {
return providerAvailable && proxyAvailable;
} catch (e) {
const error = e as Error;
if (e instanceof Redis.ReplyError) {
if (e instanceof ReplyError) {
logger.error(error, "redis");
return false;
}
@ -230,7 +264,7 @@ class NetworkDelegate {
clearTimeout(timeout);
return await response.json() as R;
return (await response.json()) as R;
} catch (e) {
throw new NetSchedulerError("Fetch error", "FETCH_ERROR", e);
}
@ -238,29 +272,26 @@ class NetworkDelegate {
private async alicloudFcRequest<R>(url: string, region: string): Promise<R> {
try {
const decoder = new TextDecoder();
const output = await new Deno.Command("aliyun", {
args: [
"fc",
"POST",
`/2023-03-30/functions/proxy-${region}/invocations`,
"--qualifier",
"LATEST",
"--header",
"Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;",
"--body",
JSON.stringify({ url: url }),
"--retry-count",
"5",
"--read-timeout",
"30",
"--connect-timeout",
"10",
"--profile",
`CVSA-${region}`,
],
}).output();
const out = decoder.decode(output.stdout);
const output = await spawnPromise("aliyun", [
"fc",
"POST",
`/2023-03-30/functions/proxy-${region}/invocations`,
"--qualifier",
"LATEST",
"--header",
"Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;",
"--body",
JSON.stringify({ url: url }),
"--retry-count",
"5",
"--read-timeout",
"30",
"--connect-timeout",
"10",
"--profile",
`CVSA-${region}`,
]);
const out = output.stdout;
const rawData = JSON.parse(out);
if (rawData.statusCode !== 200) {
// noinspection ExceptionCaughtLocallyJS
@ -269,11 +300,15 @@ class NetworkDelegate {
"ALICLOUD_PROXY_ERR",
);
} else {
return JSON.parse(JSON.parse(rawData.body)) as R;
return JSON.parse(rawData.body) as R;
}
} catch (e) {
logger.error(e as Error, "net", "fn:alicloudFcRequest");
throw new NetSchedulerError(`Unhandled error: Cannot proxy ${url} to ali-fc-${region}.`, "ALICLOUD_PROXY_ERR", e);
throw new NetSchedulerError(
`Unhandled error: Cannot proxy ${url} to ali-fc-${region}.`,
"ALICLOUD_PROXY_ERR",
e,
);
}
}
}
@ -358,7 +393,11 @@ for (const region of regions) {
}
networkDelegate.addTask("getVideoInfo", "bilibili", "all");
networkDelegate.addTask("getLatestVideos", "bilibili", "all");
networkDelegate.addTask("snapshotMilestoneVideo", "bilibili", regions.map((region) => `alicloud-${region}`));
networkDelegate.addTask(
"snapshotMilestoneVideo",
"bilibili",
regions.map((region) => `alicloud-${region}`),
);
networkDelegate.addTask("snapshotVideo", "bili_test", [
"alicloud-qingdao",
"alicloud-shanghai",

View File

@ -1,6 +1,6 @@
import networkDelegate from "@core/net/delegate.ts";
import { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d.ts";
import logger from "log/logger.ts";
import type { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d.ts";
import logger from "@core/log/logger.ts";
/*
* Fetch video metadata from bilibili API

View File

@ -0,0 +1,13 @@
{
"name": "core",
"dependencies": {
"chalk": "^5.4.1",
"ioredis": "^5.6.1",
"logform": "^2.7.0",
"postgres": "^3.4.5",
"winston": "^3.17.0"
},
"devDependencies": {
"@types/ioredis": "^5.0.0"
}
}

View File

@ -0,0 +1,18 @@
{
"include": ["**/*.ts"],
"compilerOptions": {
"baseUrl": ".",
"target": "ESNext",
"module": "ESNext",
"useDefineForClassFields": true,
"moduleResolution": "bundler",
"lib": ["ES2020", "DOM", "DOM.Iterable"],
"skipLibCheck": true,
"paths": {
"@core/*": ["./*"]
},
"allowSyntheticDefaultImports": true,
"allowImportingTsExtensions": true,
"noEmit": true
}
}

View File

@ -1,5 +1,5 @@
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { getVideoInfo } from "net/getVideoInfo.ts";
import { getVideoInfo } from "@core/net/getVideoInfo.ts";
import logger from "log/logger.ts";
export interface SnapshotNumber {

View File

@ -20,5 +20,5 @@ pnpm-debug.log*
# macOS-specific files
.DS_Store
# jetbrains setting folder
.idea/
# Docker compose
docker-compose.yml

View File

@ -6,11 +6,11 @@
"dependencies": {
"@astrojs/node": "^9.1.3",
"@astrojs/svelte": "^7.0.9",
"@astrojs/tailwind": "^6.0.2",
"@tailwindcss/vite": "^4.1.4",
"argon2id": "^1.0.1",
"astro": "^5.5.5",
"autoprefixer": "^10.4.21",
"date-fns": "^4.1.0",
"pg": "^8.11.11",
"postcss": "^8.5.3",
"svelte": "^5.25.7",
@ -39,8 +39,6 @@
"@astrojs/svelte": ["@astrojs/svelte@7.0.9", "", { "dependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.3", "svelte2tsx": "^0.7.35", "vite": "^6.2.4" }, "peerDependencies": { "astro": "^5.0.0", "svelte": "^5.1.16", "typescript": "^5.3.3" } }, "sha512-EpJfDh7eelYEj/zSwgSHdqJCx6YjiZmpVDEiNjxhnrBwM6Ll7hjllTrNQyfnv7KgJwaVo2SOSz6d1MwV52/T/w=="],
"@astrojs/tailwind": ["@astrojs/tailwind@6.0.2", "", { "dependencies": { "autoprefixer": "^10.4.21", "postcss": "^8.5.3", "postcss-load-config": "^4.0.2" }, "peerDependencies": { "astro": "^3.0.0 || ^4.0.0 || ^5.0.0", "tailwindcss": "^3.0.24" } }, "sha512-j3mhLNeugZq6A8dMNXVarUa8K6X9AW+QHU9u3lKNrPLMHhOQ0S7VeWhHwEeJFpEK1BTKEUY1U78VQv2gN6hNGg=="],
"@astrojs/telemetry": ["@astrojs/telemetry@3.2.0", "", { "dependencies": { "ci-info": "^4.1.0", "debug": "^4.3.7", "dlv": "^1.1.3", "dset": "^3.1.4", "is-docker": "^3.0.0", "is-wsl": "^3.1.0", "which-pm-runs": "^1.1.0" } }, "sha512-wxhSKRfKugLwLlr4OFfcqovk+LIFtKwLyGPqMsv+9/ibqqnW3Gv7tBhtKEb0gAyUAC4G9BTVQeQahqnQAhd6IQ=="],
"@babel/helper-string-parser": ["@babel/helper-string-parser@7.25.9", "", {}, "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA=="],
@ -349,6 +347,8 @@
"cssesc": ["cssesc@3.0.0", "", { "bin": { "cssesc": "bin/cssesc" } }, "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg=="],
"date-fns": ["date-fns@4.1.0", "", {}, "sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg=="],
"debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="],
"decode-named-character-reference": ["decode-named-character-reference@1.1.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w=="],
@ -543,8 +543,6 @@
"lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.29.2", "", { "os": "win32", "cpu": "x64" }, "sha512-EdIUW3B2vLuHmv7urfzMI/h2fmlnOQBk1xlsDxkN1tCWKjNFjfLhGxYk8C8mzpSfr+A6jFFIi8fU6LbQGsRWjA=="],
"lilconfig": ["lilconfig@3.1.3", "", {}, "sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw=="],
"locate-character": ["locate-character@3.0.0", "", {}, "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="],
"longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="],
@ -719,8 +717,6 @@
"postcss": ["postcss@8.5.3", "", { "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A=="],
"postcss-load-config": ["postcss-load-config@4.0.2", "", { "dependencies": { "lilconfig": "^3.0.0", "yaml": "^2.3.4" }, "peerDependencies": { "postcss": ">=8.0.9", "ts-node": ">=9.0.0" }, "optionalPeers": ["postcss", "ts-node"] }, "sha512-bSVhyJGL00wMVoPUzAVAnbEoWyqRxkjv64tUl427SKnPrENtq6hJwUojroMz2VB+Q1edmi4IfrAPpami5VVgMQ=="],
"postcss-value-parser": ["postcss-value-parser@4.2.0", "", {}, "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="],
"postgres-array": ["postgres-array@3.0.4", "", {}, "sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ=="],

View File

@ -1,8 +0,0 @@
{
"name": "@cvsa/frontend",
"tasks": {
"preview": "bun run astro preview",
"build": "bun run astro build"
},
"exports": "./main.ts"
}

View File

@ -11,13 +11,14 @@
"dependencies": {
"@astrojs/node": "^9.1.3",
"@astrojs/svelte": "^7.0.9",
"@astrojs/tailwind": "^6.0.2",
"@tailwindcss/vite": "^4.1.4",
"argon2id": "^1.0.1",
"astro": "^5.5.5",
"autoprefixer": "^10.4.21",
"date-fns": "^4.1.0",
"pg": "^8.11.11",
"postcss": "^8.5.3",
"postgres": "^3.4.5",
"svelte": "^5.25.7",
"tailwindcss": "^4.1.4",
"ua-parser-js": "^2.0.3",

View File

@ -0,0 +1,8 @@
import { sql } from "@core/db/dbNew";
export async function aidExists(aid: number) {
const res = await sql`
SELECT 1 FROM bilibili_metadata WHERE aid = ${aid}
`;
return res.length > 0;
}

View File

@ -0,0 +1,15 @@
import { sql } from "@core/db/dbNew";
export async function getAidFromBV(bv: string) {
const res = await sql`
SELECT aid FROM bilibili_metadata WHERE bvid = ${bv}
`
if (res.length <= 0) {
return null;
}
const row = res[0];
if (row && row.aid) {
return Number(row.aid);
}
return null;
}

View File

@ -0,0 +1,15 @@
import { sql } from "@core/db/dbNew";
export async function getVideoMetadata(aid: number) {
const res = await sql`
SELECT * FROM bilibili_metadata WHERE aid = ${aid}
`;
if (res.length <= 0) {
return null;
}
const row = res[0];
if (row) {
return row;
}
return {};
}

View File

@ -0,0 +1,11 @@
import { sql } from "@core/db/dbNew";
export async function getAllSnapshots(aid: number) {
const res = await sql`
SELECT * FROM video_snapshot WHERE aid = ${aid} ORDER BY created_at DESC
`;
if (res.length <= 0) {
return null;
}
return res;
}

View File

@ -1,71 +1,31 @@
---
import Layout from "@layouts/Layout.astro";
import TitleBar from "@components/TitleBar.astro";
import pg from "pg";
import { format } from 'date-fns';
import { zhCN } from 'date-fns/locale';
import { format } from "date-fns";
import { zhCN } from "date-fns/locale";
import MetadataRow from "@components/InfoPage/MetadataRow.astro";
import { getAllSnapshots } from "src/db/snapshots/getAllSnapshots";
import { getAidFromBV } from "src/db/bilibili_metadata/getAidFromBV";
import { getVideoMetadata } from "src/db/bilibili_metadata/getVideoMetadata";
import { aidExists as idExists } from "src/db/bilibili_metadata/aidExists";
const databaseHost = import.meta.env.DB_HOST
const databaseName = import.meta.env.DB_NAME
const databaseUser = import.meta.env.DB_USER
const databasePassword = import.meta.env.DB_PASSWORD
const databasePort = import.meta.env.DB_PORT
const databaseHost = import.meta.env.DB_HOST;
const databaseName = import.meta.env.DB_NAME;
const databaseUser = import.meta.env.DB_USER;
const databasePassword = import.meta.env.DB_PASSWORD;
const databasePort = import.meta.env.DB_PORT;
const postgresConfig = {
hostname: databaseHost,
port: parseInt(databasePort!),
database: databaseName,
user: databaseUser,
password: databasePassword,
hostname: databaseHost,
port: parseInt(databasePort!),
database: databaseName,
user: databaseUser,
password: databasePassword,
};
// 路由参数
console.log(postgresConfig);
const { id } = Astro.params;
const { Client } = pg;
const client = new Client(postgresConfig);
await client.connect();
// 数据库查询函数
async function getVideoMetadata(aid: number) {
const res = await client.query("SELECT * FROM bilibili_metadata WHERE aid = $1", [aid]);
if (res.rows.length <= 0) {
return null;
}
const row = res.rows[0];
if (row) {
return row;
}
return {};
}
async function getVideoSnapshots(aid: number) {
const res = await client.query("SELECT * FROM video_snapshot WHERE aid = $1 ORDER BY created_at DESC", [
aid,
]);
if (res.rows.length <= 0) {
return null;
}
return res.rows;
}
async function getAidFromBV(bv: string) {
const res = await client.query("SELECT aid FROM bilibili_metadata WHERE bvid = $1", [bv]);
if (res.rows.length <= 0) {
return null;
}
const row = res.rows[0];
if (row && row.aid) {
return Number(row.aid);
}
return null;
}
async function idExists(aid: number) {
const res = await client.query("SELECT COUNT(*) FROM bilibili_metadata WHERE aid = $1", [aid]);
return res.rows[0].count > 0;
}
async function getVideoAid(id: string) {
if (id.startsWith("av")) {
@ -76,27 +36,22 @@ async function getVideoAid(id: string) {
return parseInt(id);
}
// 获取数据
if (!id) {
Astro.response.status = 404;
client.end();
return new Response(null, { status: 404 });
}
const aid = await getVideoAid(id);
if (!aid || isNaN(aid)) {
Astro.response.status = 404;
client.end();
return new Response(null, { status: 404 });
}
const aidExists = await idExists(aid);
if (!aidExists) {
Astro.response.status = 404;
client.end();
return new Response(null, { status: 404 });
}
const videoInfo = await getVideoMetadata(aid);
const snapshots = await getVideoSnapshots(aid);
client.end();
const snapshots = await getAllSnapshots(aid);
interface Snapshot {
created_at: Date;
@ -115,24 +70,36 @@ interface Snapshot {
<TitleBar />
<main class="flex flex-col items-center min-h-screen gap-8 mt-10 md:mt-6 relative z-0 overflow-x-auto pb-8">
<div class="w-full lg:max-w-4xl lg:mx-auto lg:p-6">
<h1 class="text-2xl font-medium ml-2 mb-4">视频信息: <a href={`https://www.bilibili.com/video/av${aid}`} class="underline ">av{aid}</a></h1>
<h1 class="text-2xl font-medium ml-2 mb-4">
视频信息: <a href={`https://www.bilibili.com/video/av${aid}`} class="underline">av{aid}</a>
</h1>
<div class="mb-6">
<h2 class="px-2 mb-2 text-xl font-medium">基本信息</h2>
<div class="overflow-x-auto max-w-full px-2">
<table class="table-fixed">
<tbody>
<MetadataRow title={id} description={videoInfo?.id}/>
<MetadataRow title={videoInfo?.aid} description={videoInfo?.aid}/>
<MetadataRow title={videoInfo?.bvid} description={videoInfo?.bvid}/>
<MetadataRow title="标题" description={videoInfo?.title}/>
<MetadataRow title="描述" description={videoInfo?.description}/>
<MetadataRow title="UID" description={videoInfo?.uid}/>
<MetadataRow title="标签" description={videoInfo?.tags}/>
<MetadataRow title="发布时间" description={format(new Date(videoInfo?.pubdate), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN })}/>
<MetadataRow title="时长 (秒)" description={videoInfo?.duration}/>
<MetadataRow title="创建时间" description={format(new Date(videoInfo?.created_at), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN })}/>
<MetadataRow title="封面" description={videoInfo?.cover_url}/>
<MetadataRow title={id} description={videoInfo?.id} />
<MetadataRow title={videoInfo?.aid} description={videoInfo?.aid} />
<MetadataRow title={videoInfo?.bvid} description={videoInfo?.bvid} />
<MetadataRow title="标题" description={videoInfo?.title} />
<MetadataRow title="描述" description={videoInfo?.description} />
<MetadataRow title="UID" description={videoInfo?.uid} />
<MetadataRow title="标签" description={videoInfo?.tags} />
<MetadataRow
title="发布时间"
description={format(new Date(videoInfo?.published_at), "yyyy-MM-dd HH:mm:ss", {
locale: zhCN,
})}
/>
<MetadataRow title="时长 (秒)" description={videoInfo?.duration} />
<MetadataRow
title="创建时间"
description={format(new Date(videoInfo?.created_at), "yyyy-MM-dd HH:mm:ss", {
locale: zhCN,
})}
/>
<MetadataRow title="封面" description={videoInfo?.cover_url} />
</tbody>
</table>
</div>
@ -140,40 +107,46 @@ interface Snapshot {
<div>
<h2 class="px-2 mb-2 text-xl font-medium">播放量历史数据</h2>
{snapshots && snapshots.length > 0 ? (
<div class="overflow-x-auto px-2">
<table class="table-auto w-full">
<thead>
<tr>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">创建时间</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">观看</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">硬币</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">点赞</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">收藏</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">分享</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">弹幕</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">评论</th>
</tr>
</thead>
<tbody>
{snapshots.map((snapshot: Snapshot) => (
{
snapshots && snapshots.length > 0 ? (
<div class="overflow-x-auto px-2">
<table class="table-auto w-full">
<thead>
<tr>
<td class="border dark:border-zinc-500 px-4 py-2">{format(new Date(snapshot.created_at), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN })}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.views}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.coins}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.likes}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.favorites}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.shares}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.danmakus}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.replies}</td>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">创建时间</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">观看</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">硬币</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">点赞</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">收藏</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">分享</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">弹幕</th>
<th class="border dark:border-zinc-500 px-4 py-2 font-medium">评论</th>
</tr>
))}
</tbody>
</table>
</div>
) : (
<p>暂无历史数据。</p>
)}
</thead>
<tbody>
{snapshots.map((snapshot: Snapshot) => (
<tr>
<td class="border dark:border-zinc-500 px-4 py-2">
{format(new Date(snapshot.created_at), "yyyy-MM-dd HH:mm:ss", {
locale: zhCN,
})}
</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.views}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.coins}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.likes}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.favorites}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.shares}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.danmakus}</td>
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.replies}</td>
</tr>
))}
</tbody>
</table>
</div>
) : (
<p>暂无历史数据。</p>
)
}
</div>
</div>
</main>

View File

@ -11,6 +11,7 @@
"@assets/*": ["src/assets/*"],
"@styles": ["src/styles/*"],
"@core/*": ["../core/*"]
}
},
"verbatimModuleSyntax": true
}
}