From 59f09ca5ebdd13eb8e2409ad280e161430ff31a7 Mon Sep 17 00:00:00 2001 From: alikia2x Date: Mon, 5 May 2025 01:53:33 +0800 Subject: [PATCH] ref: switch to bun --- .gitignore | 9 +- .idea/cvsa.iml | 2 + .prettierrc | 8 + bun.lock | 177 ++++++- package.json | 8 +- packages/core/db/dbNew.ts | 2 + packages/core/db/schema.d.ts | 8 +- packages/core/log/test.ts | 2 +- packages/core/net/delegate.ts | 2 +- packages/crawler/.gitignore | 2 + packages/crawler/bun.lock | 433 ++++++++++++++++++ packages/crawler/db/bilibili_metadata.ts | 132 +++--- packages/crawler/db/init.ts | 6 - packages/crawler/db/snapshot.ts | 25 +- packages/crawler/db/snapshotSchedule.ts | 279 ++++++----- packages/crawler/db/songs.ts | 72 ++- packages/crawler/db/withConnection.ts | 32 -- packages/crawler/deno.json | 47 -- packages/crawler/global.d.ts | 3 + packages/crawler/ml/akari.ts | 10 +- packages/crawler/ml/const.ts | 1 + packages/crawler/ml/manager.ts | 4 +- packages/crawler/mq/exec/archiveSnapshots.ts | 28 +- packages/crawler/mq/exec/classifyVideo.ts | 23 +- packages/crawler/mq/exec/collectSongs.ts | 12 +- .../mq/exec/dispatchMilestoneSnapshots.ts | 24 +- .../mq/exec/dispatchRegularSnapshots.ts | 32 +- packages/crawler/mq/exec/getLatestVideos.ts | 10 +- packages/crawler/mq/exec/getVideoInfo.ts | 24 +- packages/crawler/mq/exec/scheduleCleanup.ts | 38 +- packages/crawler/mq/exec/snapshotTick.ts | 22 +- packages/crawler/mq/exec/snapshotVideo.ts | 53 +-- packages/crawler/mq/exec/takeBulkSnapshot.ts | 48 +- packages/crawler/mq/init.ts | 103 +++-- packages/crawler/mq/scheduling.ts | 12 +- packages/crawler/mq/task/collectSongs.ts | 40 +- .../mq/task/getTimeoutSchedulesCount.ts | 16 +- packages/crawler/mq/task/getVideoDetails.ts | 57 +-- packages/crawler/mq/task/getVideoStats.ts | 16 +- packages/crawler/mq/task/queueLatestVideo.ts | 12 +- .../mq/task/regularSnapshotInterval.ts | 12 +- .../mq/task/removeAllTimeoutSchedules.ts | 17 +- packages/crawler/net/bulkGetVideoStats.ts | 4 +- packages/crawler/net/getLatestVideoAids.ts | 4 +- packages/crawler/net/getVideoDetails.ts | 4 +- packages/crawler/package.json | 24 + packages/crawler/src/build.ts | 15 + packages/crawler/src/filterWorker.ts | 21 +- packages/crawler/src/worker.ts | 21 +- .../crawler/test/db/snapshotSchedule.test.ts | 120 +++++ packages/crawler/tsconfig.json | 18 + packages/crawler/vitest.config.ts | 6 + 52 files changed, 1397 insertions(+), 703 deletions(-) create mode 100644 .prettierrc create mode 100644 packages/crawler/.gitignore create mode 100644 packages/crawler/bun.lock delete mode 100644 packages/crawler/db/init.ts delete mode 100644 packages/crawler/db/withConnection.ts delete mode 100644 packages/crawler/deno.json create mode 100644 packages/crawler/global.d.ts create mode 100644 packages/crawler/ml/const.ts create mode 100644 packages/crawler/package.json create mode 100644 packages/crawler/src/build.ts create mode 100644 packages/crawler/test/db/snapshotSchedule.test.ts create mode 100644 packages/crawler/tsconfig.json create mode 100644 packages/crawler/vitest.config.ts diff --git a/.gitignore b/.gitignore index c0490d2..a9cdf30 100644 --- a/.gitignore +++ b/.gitignore @@ -63,10 +63,7 @@ package-lock.json # dotenv environment variable files .env -.env.development.local -.env.test.local -.env.production.local -.env.local +.env.* # npm dependencies node_modules/ @@ -94,4 +91,8 @@ model/ data/ redis/ +# Build +dist/ +build/ + docker-compose.yml \ No newline at end of file diff --git a/.idea/cvsa.iml b/.idea/cvsa.iml index 78a2917..7bfcf20 100644 --- a/.idea/cvsa.iml +++ b/.idea/cvsa.iml @@ -26,6 +26,8 @@ + + diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..0961adb --- /dev/null +++ b/.prettierrc @@ -0,0 +1,8 @@ +{ + "useTabs": true, + "tabWidth": 4, + "trailingComma": "none", + "singleQuote": false, + "printWidth": 120, + "endOfLine": "lf" +} diff --git a/bun.lock b/bun.lock index 11297e3..d6a4816 100644 --- a/bun.lock +++ b/bun.lock @@ -6,6 +6,11 @@ "dependencies": { "postgres": "^3.4.5", }, + "devDependencies": { + "vite-tsconfig-paths": "^5.1.4", + "vitest": "^3.1.2", + "vitest-tsconfig-paths": "^3.4.1", + }, }, "packages/backend": { "name": "backend", @@ -91,6 +96,8 @@ "@colors/colors": ["@colors/colors@1.6.0", "", {}, "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA=="], + "@cush/relative": ["@cush/relative@1.0.0", "", {}, "sha512-RpfLEtTlyIxeNPGKcokS+p3BZII/Q3bYxryFRglh5H3A3T8q9fsLYm72VYAMEOOIBLEa8o93kFLiBDUWKrwXZA=="], + "@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="], "@emnapi/runtime": ["@emnapi/runtime@1.4.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ=="], @@ -185,6 +192,8 @@ "@ioredis/commands": ["@ioredis/commands@1.2.0", "", {}, "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="], + "@isaacs/cliui": ["@isaacs/cliui@8.0.2", "", { "dependencies": { "string-width": "^5.1.2", "string-width-cjs": "npm:string-width@^4.2.0", "strip-ansi": "^7.0.1", "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", "wrap-ansi": "^8.1.0", "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="], + "@jridgewell/gen-mapping": ["@jridgewell/gen-mapping@0.3.8", "", { "dependencies": { "@jridgewell/set-array": "^1.2.1", "@jridgewell/sourcemap-codec": "^1.4.10", "@jridgewell/trace-mapping": "^0.3.24" } }, "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA=="], "@jridgewell/resolve-uri": ["@jridgewell/resolve-uri@3.1.2", "", {}, "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw=="], @@ -197,6 +206,8 @@ "@oslojs/encoding": ["@oslojs/encoding@1.1.0", "", {}, "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ=="], + "@pkgjs/parseargs": ["@pkgjs/parseargs@0.11.0", "", {}, "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg=="], + "@rabbit-company/argon2id": ["@rabbit-company/argon2id@2.1.0", "", { "peerDependencies": { "typescript": "^5.6.2" } }, "sha512-X/kt89qjmS9+Zh+DYCGcWeTwHa4C8vY8T3EnSma+vWj7spMzAYX4F8vmGUkny9hygpTOeC/yXwAUdJAfJ52H+w=="], "@rollup/plugin-wasm": ["@rollup/plugin-wasm@6.2.2", "", { "dependencies": { "@rollup/pluginutils": "^5.0.2" }, "peerDependencies": { "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" }, "optionalPeers": ["rollup"] }, "sha512-gpC4R1G9Ni92ZIRTexqbhX7U+9estZrbhP+9SRb0DW9xpB9g7j34r+J2hqrcW/lRI7dJaU84MxZM0Rt82tqYPQ=="], @@ -305,6 +316,8 @@ "@types/ioredis": ["@types/ioredis@5.0.0", "", { "dependencies": { "ioredis": "*" } }, "sha512-zJbJ3FVE17CNl5KXzdeSPtdltc4tMT3TzC6fxQS0sQngkbFZ6h+0uTafsRqu+eSLIugf6Yb0Ea0SUuRr42Nk9g=="], + "@types/json5": ["@types/json5@0.0.29", "", {}, "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ=="], + "@types/mdast": ["@types/mdast@4.0.4", "", { "dependencies": { "@types/unist": "*" } }, "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA=="], "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="], @@ -323,6 +336,20 @@ "@ungap/structured-clone": ["@ungap/structured-clone@1.3.0", "", {}, "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g=="], + "@vitest/expect": ["@vitest/expect@3.1.2", "", { "dependencies": { "@vitest/spy": "3.1.2", "@vitest/utils": "3.1.2", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" } }, "sha512-O8hJgr+zREopCAqWl3uCVaOdqJwZ9qaDwUP7vy3Xigad0phZe9APxKhPcDNqYYi0rX5oMvwJMSCAXY2afqeTSA=="], + + "@vitest/mocker": ["@vitest/mocker@3.1.2", "", { "dependencies": { "@vitest/spy": "3.1.2", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^5.0.0 || ^6.0.0" }, "optionalPeers": ["msw", "vite"] }, "sha512-kOtd6K2lc7SQ0mBqYv/wdGedlqPdM/B38paPY+OwJ1XiNi44w3Fpog82UfOibmHaV9Wod18A09I9SCKLyDMqgw=="], + + "@vitest/pretty-format": ["@vitest/pretty-format@3.1.2", "", { "dependencies": { "tinyrainbow": "^2.0.0" } }, "sha512-R0xAiHuWeDjTSB3kQ3OQpT8Rx3yhdOAIm/JM4axXxnG7Q/fS8XUwggv/A4xzbQA+drYRjzkMnpYnOGAc4oeq8w=="], + + "@vitest/runner": ["@vitest/runner@3.1.2", "", { "dependencies": { "@vitest/utils": "3.1.2", "pathe": "^2.0.3" } }, "sha512-bhLib9l4xb4sUMPXnThbnhX2Yi8OutBMA8Yahxa7yavQsFDtwY/jrUZwpKp2XH9DhRFJIeytlyGpXCqZ65nR+g=="], + + "@vitest/snapshot": ["@vitest/snapshot@3.1.2", "", { "dependencies": { "@vitest/pretty-format": "3.1.2", "magic-string": "^0.30.17", "pathe": "^2.0.3" } }, "sha512-Q1qkpazSF/p4ApZg1vfZSQ5Yw6OCQxVMVrLjslbLFA1hMDrT2uxtqMaw8Tc/jy5DLka1sNs1Y7rBcftMiaSH/Q=="], + + "@vitest/spy": ["@vitest/spy@3.1.2", "", { "dependencies": { "tinyspy": "^3.0.2" } }, "sha512-OEc5fSXMws6sHVe4kOFyDSj/+4MSwst0ib4un0DlcYgQvRuYQ0+M2HyqGaauUMnjq87tmUaMNDxKQx7wNfVqPA=="], + + "@vitest/utils": ["@vitest/utils@3.1.2", "", { "dependencies": { "@vitest/pretty-format": "3.1.2", "loupe": "^3.1.3", "tinyrainbow": "^2.0.0" } }, "sha512-5GGd0ytZ7BH3H6JTj9Kw7Prn1Nbg0wZVrIvou+UWxm54d+WoXXgAgjFJ8wn3LdagWLFSEfpPeyYrByZaGEZHLg=="], + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], "acorn": ["acorn@8.14.1", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-OvQ/2pUDKmgfCg++xsTX1wGxfTaszcHVcTctW4UJB4hibJx2HXxxO5UmVgyjMa+ZDsiaf5wWLXYpRWMmBI0QHg=="], @@ -333,6 +360,8 @@ "ansi-styles": ["ansi-styles@6.2.1", "", {}, "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug=="], + "any-promise": ["any-promise@1.3.0", "", {}, "sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A=="], + "anymatch": ["anymatch@3.1.3", "", { "dependencies": { "normalize-path": "^3.0.0", "picomatch": "^2.0.4" } }, "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw=="], "argon2id": ["argon2id@1.0.1", "", {}, "sha512-rsiD3lX+0L0CsiZARp3bf9EGxprtuWAT7PpiJd+Fk53URV0/USOQkBIP1dLTV8t6aui0ECbymQ9W9YCcTd6XgA=="], @@ -343,6 +372,8 @@ "array-iterate": ["array-iterate@2.0.1", "", {}, "sha512-I1jXZMjAgCMmxT4qxXfPXa6SthSoE8h6gkSI9BGGNv8mP8G/v0blc+qFnZu6K42vTOiuME596QaLO0TP3Lk0xg=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], + "astro": ["astro@5.7.8", "", { "dependencies": { "@astrojs/compiler": "^2.11.0", "@astrojs/internal-helpers": "0.6.1", "@astrojs/markdown-remark": "6.3.1", "@astrojs/telemetry": "3.2.1", "@capsizecss/unpack": "^2.4.0", "@oslojs/encoding": "^1.1.0", "@rollup/pluginutils": "^5.1.4", "acorn": "^8.14.1", "aria-query": "^5.3.2", "axobject-query": "^4.1.0", "boxen": "8.0.1", "ci-info": "^4.2.0", "clsx": "^2.1.1", "common-ancestor-path": "^1.0.1", "cookie": "^1.0.2", "cssesc": "^3.0.0", "debug": "^4.4.0", "deterministic-object-hash": "^2.0.2", "devalue": "^5.1.1", "diff": "^5.2.0", "dlv": "^1.1.3", "dset": "^3.1.4", "es-module-lexer": "^1.6.0", "esbuild": "^0.25.0", "estree-walker": "^3.0.3", "flattie": "^1.1.1", "github-slugger": "^2.0.0", "html-escaper": "3.0.3", "http-cache-semantics": "^4.1.1", "js-yaml": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.17", "magicast": "^0.3.5", "mrmime": "^2.0.1", "neotraverse": "^0.6.18", "p-limit": "^6.2.0", "p-queue": "^8.1.0", "package-manager-detector": "^1.1.0", "picomatch": "^4.0.2", "prompts": "^2.4.2", "rehype": "^13.0.2", "semver": "^7.7.1", "shiki": "^3.2.1", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.12", "tsconfck": "^3.1.5", "ultrahtml": "^1.6.0", "unifont": "~0.4.1", "unist-util-visit": "^5.0.0", "unstorage": "^1.15.0", "vfile": "^6.0.3", "vite": "^6.2.6", "vitefu": "^1.0.6", "xxhash-wasm": "^1.1.0", "yargs-parser": "^21.1.1", "yocto-spinner": "^0.2.1", "zod": "^3.24.2", "zod-to-json-schema": "^3.24.5", "zod-to-ts": "^1.2.0" }, "optionalDependencies": { "sharp": "^0.33.3" }, "bin": { "astro": "astro.js" } }, "sha512-82ku6+wOGXP5c5+YOkBzEGJ/k8/GVSeN0xD/TNUrPf5nvWpGGpngxtUAwuruKF6wIxRC1/SwlUVCs/4QT98iZQ=="], "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], @@ -357,6 +388,8 @@ "bail": ["bail@2.0.2", "", {}, "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw=="], + "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + "base-64": ["base-64@1.0.0", "", {}, "sha512-kwDPIFCGx0NZHog36dj+tHiwP4QMzsZ3AgMViUBKI0+V5n4U0ufTCUMhnQ04diaRI8EX/QcPfql7zlhZ7j4zgg=="], "base64-js": ["base64-js@1.5.1", "", {}, "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="], @@ -367,6 +400,8 @@ "boxen": ["boxen@8.0.1", "", { "dependencies": { "ansi-align": "^3.0.1", "camelcase": "^8.0.0", "chalk": "^5.3.0", "cli-boxes": "^3.0.0", "string-width": "^7.2.0", "type-fest": "^4.21.0", "widest-line": "^5.0.0", "wrap-ansi": "^9.0.0" } }, "sha512-F3PH5k5juxom4xktynS7MoFY+NUWH5LC4CnH11YB8NPew+HLpmBLCybSAEyb2F+4pRXhuhWqFesoQd6DAyc2hw=="], + "brace-expansion": ["brace-expansion@2.0.1", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="], + "brotli": ["brotli@1.3.3", "", { "dependencies": { "base64-js": "^1.1.2" } }, "sha512-oTKjJdShmDuGW94SyyaoQvAjf30dZaHnjJ8uAF+u2/vGJkJbJPJAT1gDiOJP5v1Zb6f9KEyW/1HpuaWIXtGHPg=="], "browserslist": ["browserslist@4.24.4", "", { "dependencies": { "caniuse-lite": "^1.0.30001688", "electron-to-chromium": "^1.5.73", "node-releases": "^2.0.19", "update-browserslist-db": "^1.1.1" }, "bin": { "browserslist": "cli.js" } }, "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A=="], @@ -375,6 +410,8 @@ "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], + "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], @@ -385,6 +422,8 @@ "ccount": ["ccount@2.0.1", "", {}, "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg=="], + "chai": ["chai@5.2.0", "", { "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", "deep-eql": "^5.0.1", "loupe": "^3.1.0", "pathval": "^2.0.0" } }, "sha512-mCuXncKXk5iCLhfhwTc0izo0gtEmpz5CtG2y8GiOINBlMVS6v8TMRc5TaLWKS6692m9+dVVfzgeVxR5UxWHTYw=="], + "chalk": ["chalk@5.4.1", "", {}, "sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w=="], "character-entities": ["character-entities@2.0.2", "", {}, "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ=="], @@ -393,6 +432,8 @@ "character-entities-legacy": ["character-entities-legacy@3.0.0", "", {}, "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ=="], + "check-error": ["check-error@2.1.1", "", {}, "sha512-OAlb+T7V4Op9OwdkjmguYRqncdlx5JiofwOAUkmTF+jNdHwzTaTs4sRAGpzLF3oOz5xAyDGrPgeIDFQmDOTiJw=="], + "chokidar": ["chokidar@4.0.3", "", { "dependencies": { "readdirp": "^4.0.1" } }, "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA=="], "ci-info": ["ci-info@4.2.0", "", {}, "sha512-cYY9mypksY8NRqgDB1XD1RiJL338v/551niynFTGkZOO2LHuB2OmOYxDIe/ttN9AHwrqdum1360G3ald0W9kCg=="], @@ -419,6 +460,8 @@ "comma-separated-tokens": ["comma-separated-tokens@2.0.3", "", {}, "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg=="], + "commander": ["commander@4.1.1", "", {}, "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA=="], + "common-ancestor-path": ["common-ancestor-path@1.0.1", "", {}, "sha512-L3sHRo1pXXEqX8VU28kfgUY+YGsk09hPqZiZmLacNib6XNTCM8ubYeT7ryXQw8asB1sKgcU5lkB7ONug08aB8w=="], "content-disposition": ["content-disposition@1.0.0", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg=="], @@ -435,6 +478,8 @@ "cross-fetch": ["cross-fetch@3.2.0", "", { "dependencies": { "node-fetch": "^2.7.0" } }, "sha512-Q+xVJLoGOeIMXZmbUK4HYk+69cQH6LudR0Vu/pRm2YlU/hDV9CiS0gKUMaWY5f2NeUH9C1nV3bsTlCo0FsTV1Q=="], + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + "crossws": ["crossws@0.3.4", "", { "dependencies": { "uncrypto": "^0.1.3" } }, "sha512-uj0O1ETYX1Bh6uSgktfPvwDiPYGQ3aI4qVsaC/LWpkIzGj1nUYm5FK3K+t11oOlpN01lGbprFCH4wBlKdJjVgw=="], "css-tree": ["css-tree@3.1.0", "", { "dependencies": { "mdn-data": "2.12.2", "source-map-js": "^1.0.1" } }, "sha512-0eW44TGN5SQXU1mWSkKwFstI/22X2bG1nYzZTYMAWjylYURhse752YgbE4Cx46AC+bAvI+/dYTPRk1LqSUnu6w=="], @@ -449,6 +494,8 @@ "dedent-js": ["dedent-js@1.0.1", "", {}, "sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ=="], + "deep-eql": ["deep-eql@5.0.2", "", {}, "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q=="], + "deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="], "defu": ["defu@6.1.4", "", {}, "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg=="], @@ -483,6 +530,8 @@ "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + "eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="], + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], "electron-to-chromium": ["electron-to-chromium@1.5.143", "", {}, "sha512-QqklJMOFBMqe46k8iIOwA9l2hz57V2OKMmP5eSWcUvwx+mASAsbU+wkF1pHjn9ZVSBPrsYWr4/W/95y5SwYg2g=="], @@ -525,6 +574,8 @@ "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], + "expect-type": ["expect-type@1.2.1", "", {}, "sha512-/kP8CAwxzLVEeFrMm4kMmy4CCDlpipyA7MYLVrdJIkV0fYF0UaigQHRsxHiuY/GEea+bh4KSv3TIlgr+2UL6bw=="], + "express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="], "express-rate-limit": ["express-rate-limit@7.5.0", "", { "peerDependencies": { "express": "^4.11 || 5 || ^5.0.0-beta.1" } }, "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg=="], @@ -545,6 +596,8 @@ "fontkit": ["fontkit@2.0.4", "", { "dependencies": { "@swc/helpers": "^0.5.12", "brotli": "^1.3.2", "clone": "^2.1.2", "dfa": "^1.2.0", "fast-deep-equal": "^3.1.3", "restructure": "^3.0.0", "tiny-inflate": "^1.0.3", "unicode-properties": "^1.4.0", "unicode-trie": "^2.0.0" } }, "sha512-syetQadaUEDNdxdugga9CpEYVaQIxOwk7GlwZWWZ19//qW4zE5bknOKeMBDYAASwnpaSHKJITRLMF9m1fp3s6g=="], + "foreground-child": ["foreground-child@3.3.1", "", { "dependencies": { "cross-spawn": "^7.0.6", "signal-exit": "^4.0.1" } }, "sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw=="], + "form-data": ["form-data@4.0.2", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "mime-types": "^2.1.12" } }, "sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], @@ -567,6 +620,10 @@ "github-slugger": ["github-slugger@2.0.0", "", {}, "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw=="], + "glob": ["glob@10.4.5", "", { "dependencies": { "foreground-child": "^3.1.0", "jackspeak": "^3.1.2", "minimatch": "^9.0.4", "minipass": "^7.1.2", "package-json-from-dist": "^1.0.0", "path-scurry": "^1.11.1" }, "bin": { "glob": "dist/esm/bin.mjs" } }, "sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg=="], + + "glob-regex": ["glob-regex@0.3.2", "", {}, "sha512-m5blUd3/OqDTWwzBBtWBPrGlAzatRywHameHeekAZyZrskYouOGdNB8T/q6JucucvJXtOuyHIn0/Yia7iDasDw=="], + "globrex": ["globrex@0.1.2", "", {}, "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -645,10 +702,16 @@ "is-wsl": ["is-wsl@3.1.0", "", { "dependencies": { "is-inside-container": "^1.0.0" } }, "sha512-UcVfVfaK4Sc4m7X3dUSoHoozQGBEFeDC+zVo06t98xe8CzHSZZBekNXH+tu0NalHolcJ/QAGqS46Hef7QXBIMw=="], + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "jackspeak": ["jackspeak@3.4.3", "", { "dependencies": { "@isaacs/cliui": "^8.0.2" }, "optionalDependencies": { "@pkgjs/parseargs": "^0.11.0" } }, "sha512-OGlZQpz2yfahA/Rd1Y8Cd9SIEsqvXkLVoSw/cgwhnhFMDbsQFeZYoJJ7bIZBS9BcamUW96asq/npPWugM+RQBw=="], + "jiti": ["jiti@2.4.2", "", { "bin": { "jiti": "lib/jiti-cli.mjs" } }, "sha512-rg9zJN+G4n2nfJl5MW3BMygZX56zKPNVEYYqq7adpmMh4Jn2QNEwhvQlFy6jPVdcod7txZtKHWnyZiA3a0zP7A=="], "js-yaml": ["js-yaml@4.1.0", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA=="], + "json5": ["json5@1.0.2", "", { "dependencies": { "minimist": "^1.2.0" }, "bin": { "json5": "lib/cli.js" } }, "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA=="], + "kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="], "kuler": ["kuler@2.0.0", "", {}, "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A=="], @@ -675,6 +738,8 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.29.2", "", { "os": "win32", "cpu": "x64" }, "sha512-EdIUW3B2vLuHmv7urfzMI/h2fmlnOQBk1xlsDxkN1tCWKjNFjfLhGxYk8C8mzpSfr+A6jFFIi8fU6LbQGsRWjA=="], + "lines-and-columns": ["lines-and-columns@1.2.4", "", {}, "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg=="], + "locate-character": ["locate-character@3.0.0", "", {}, "sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], @@ -685,6 +750,8 @@ "longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="], + "loupe": ["loupe@3.1.3", "", {}, "sha512-kkIp7XSkP78ZxJEsSxW3712C6teJVoeHHwgo9zJ380de7IYyJ2ISlxojcH2pC5OFLewESmnRi/+XCDIEEVyoug=="], + "lower-case": ["lower-case@2.0.2", "", { "dependencies": { "tslib": "^2.0.3" } }, "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg=="], "lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], @@ -789,10 +856,18 @@ "mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="], + "minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + + "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], + + "minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="], + "mrmime": ["mrmime@2.0.1", "", {}, "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + "mz": ["mz@2.7.0", "", { "dependencies": { "any-promise": "^1.0.0", "object-assign": "^4.0.1", "thenify-all": "^1.0.0" } }, "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q=="], + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], @@ -815,6 +890,8 @@ "normalize-range": ["normalize-range@0.1.2", "", {}, "sha512-bdok/XvKII3nUpklnV6P2hxtMNrCboOjAcyBuQnWEhO665FwrSNRxU+AqpsyvO6LgGYPspN+lu5CLtw4jPRKNA=="], + "object-assign": ["object-assign@4.1.1", "", {}, "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg=="], + "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], "obuf": ["obuf@1.1.2", "", {}, "sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg=="], @@ -839,6 +916,8 @@ "p-timeout": ["p-timeout@6.1.4", "", {}, "sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg=="], + "package-json-from-dist": ["package-json-from-dist@1.0.1", "", {}, "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw=="], + "package-manager-detector": ["package-manager-detector@1.2.0", "", {}, "sha512-PutJepsOtsqVfUsxCzgTTpyXmiAgvKptIgY4th5eq5UXXFhj5PxfQ9hnGkypMeovpAvVshFRItoFHYO18TCOqA=="], "pako": ["pako@0.2.9", "", {}, "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA=="], @@ -851,8 +930,16 @@ "pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="], + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "path-scurry": ["path-scurry@1.11.1", "", { "dependencies": { "lru-cache": "^10.2.0", "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" } }, "sha512-Xa4Nw17FS9ApQFJ9umLiJS4orGjm7ZzwUrwamcGQuHSzDyth9boKDaycYdDcZDuqYATXw4HFXgaqWTctW/v1HA=="], + "path-to-regexp": ["path-to-regexp@8.2.0", "", {}, "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ=="], + "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "pathval": ["pathval@2.0.0", "", {}, "sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA=="], + "pg": ["pg@8.15.6", "", { "dependencies": { "pg-connection-string": "^2.8.5", "pg-pool": "^3.9.6", "pg-protocol": "^1.9.5", "pg-types": "^2.1.0", "pgpass": "1.x" }, "optionalDependencies": { "pg-cloudflare": "^1.2.5" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-yvao7YI3GdmmrslNVsZgx9PfntfWrnXwtR+K/DjI0I/sTKif4Z623um+sjVZ1hk5670B+ODjvHDAckKdjmPTsg=="], "pg-cloudflare": ["pg-cloudflare@1.2.5", "", {}, "sha512-OOX22Vt0vOSRrdoUPKJ8Wi2OpE/o/h9T8X1s4qSkCedbNah9ei2W2765be8iMVxQUsvgT7zIAT2eIa9fs5+vtg=="], @@ -875,6 +962,8 @@ "picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="], + "pirates": ["pirates@4.0.7", "", {}, "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA=="], + "postcss": ["postcss@8.5.3", "", { "dependencies": { "nanoid": "^3.3.8", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A=="], "postcss-value-parser": ["postcss-value-parser@4.2.0", "", {}, "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="], @@ -917,6 +1006,8 @@ "readdirp": ["readdirp@4.1.2", "", {}, "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg=="], + "recrawl-sync": ["recrawl-sync@2.2.3", "", { "dependencies": { "@cush/relative": "^1.0.0", "glob-regex": "^0.3.0", "slash": "^3.0.0", "sucrase": "^3.20.3", "tslib": "^1.9.3" } }, "sha512-vSaTR9t+cpxlskkdUFrsEpnf67kSmPk66yAGT1fZPrDudxQjoMzPgQhSMImQ0pAw5k0NPirefQfhopSjhdUtpQ=="], + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], @@ -977,6 +1068,10 @@ "sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + "shiki": ["shiki@3.3.0", "", { "dependencies": { "@shikijs/core": "3.3.0", "@shikijs/engine-javascript": "3.3.0", "@shikijs/engine-oniguruma": "3.3.0", "@shikijs/langs": "3.3.0", "@shikijs/themes": "3.3.0", "@shikijs/types": "3.3.0", "@shikijs/vscode-textmate": "^10.0.2", "@types/hast": "^3.0.4" } }, "sha512-j0Z1tG5vlOFGW8JVj0Cpuatzvshes7VJy5ncDmmMaYcmnGW0Js1N81TOW98ivTFNZfKRn9uwEg/aIm638o368g=="], "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], @@ -987,10 +1082,16 @@ "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], + + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + "simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="], "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], + "slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="], + "smol-toml": ["smol-toml@1.3.4", "", {}, "sha512-UOPtVuYkzYGee0Bd2Szz8d2G3RfMfJ2t3qVdZUAozZyAk+a0Sxa+QKix0YCwjL/A1RR0ar44nCxaoN9FxdJGwA=="], "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], @@ -1001,18 +1102,30 @@ "stack-trace": ["stack-trace@0.0.10", "", {}, "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg=="], + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], "statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], + "std-env": ["std-env@3.9.0", "", {}, "sha512-UGvjygr6F6tpH7o2qyqR6QYpwraIjKSdtzyBdyytFOHmPZY917kwdwLG0RbOjWOnKmnm3PeHjaoLLMie7kPLQw=="], + "string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="], + "string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + "string_decoder": ["string_decoder@1.3.0", "", { "dependencies": { "safe-buffer": "~5.2.0" } }, "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA=="], "stringify-entities": ["stringify-entities@4.0.4", "", { "dependencies": { "character-entities-html4": "^2.0.0", "character-entities-legacy": "^3.0.0" } }, "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg=="], "strip-ansi": ["strip-ansi@7.1.0", "", { "dependencies": { "ansi-regex": "^6.0.1" } }, "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ=="], + "strip-ansi-cjs": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "strip-bom": ["strip-bom@3.0.0", "", {}, "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA=="], + + "sucrase": ["sucrase@3.35.0", "", { "dependencies": { "@jridgewell/gen-mapping": "^0.3.2", "commander": "^4.0.0", "glob": "^10.3.10", "lines-and-columns": "^1.1.6", "mz": "^2.7.0", "pirates": "^4.0.1", "ts-interface-checker": "^0.1.9" }, "bin": { "sucrase": "bin/sucrase", "sucrase-node": "bin/sucrase-node" } }, "sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA=="], + "svelte": ["svelte@5.28.2", "", { "dependencies": { "@ampproject/remapping": "^2.3.0", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "esm-env": "^1.2.1", "esrap": "^1.4.6", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-FbWBxgWOpQfhKvoGJv/TFwzqb4EhJbwCD17dB0tEpQiw1XyUEKZJtgm4nA4xq3LLsMo7hu5UY/BOFmroAxKTMg=="], "svelte2tsx": ["svelte2tsx@0.7.36", "", { "dependencies": { "dedent-js": "^1.0.1", "pascal-case": "^3.1.1" }, "peerDependencies": { "svelte": "^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0", "typescript": "^4.9.4 || ^5.0.0" } }, "sha512-nBlERuCZRwmpebC8m0vDqZ9oaKsqW8frQS2l3zwFQW1voQIkItYtHxh1F5OTZEmE0meDIH6cxU36eIOQVOxlCw=="], @@ -1023,14 +1136,26 @@ "text-hex": ["text-hex@1.0.0", "", {}, "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg=="], + "thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="], + + "thenify-all": ["thenify-all@1.6.0", "", { "dependencies": { "thenify": ">= 3.1.0 < 4" } }, "sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA=="], + "tiny-case": ["tiny-case@1.0.3", "", {}, "sha512-Eet/eeMhkO6TX8mnUteS9zgPbUMQa4I6Kkp5ORiBD5476/m+PIRiumP5tmh5ioJpH7k51Kehawy2UDfsnxxY8Q=="], "tiny-inflate": ["tiny-inflate@1.0.3", "", {}, "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], + "tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], "tinyglobby": ["tinyglobby@0.2.13", "", { "dependencies": { "fdir": "^6.4.4", "picomatch": "^4.0.2" } }, "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw=="], + "tinypool": ["tinypool@1.0.2", "", {}, "sha512-al6n+QEANGFOMf/dmUMsuS5/r9B06uwlyNjZZql/zv8J7ybHCgoihBNORZCY2mzUuAnomQa2JdhyHKzZxPCrFA=="], + + "tinyrainbow": ["tinyrainbow@2.0.0", "", {}, "sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw=="], + + "tinyspy": ["tinyspy@3.0.2", "", {}, "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q=="], + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], "toposort": ["toposort@2.0.2", "", {}, "sha512-0a5EOkAUp8D4moMi2W8ZF8jcga7BgZd91O/yabJCFY8az+XSzeGyTKs0Aoo897iV1Nj6guFq8orWDS96z91oGg=="], @@ -1043,9 +1168,13 @@ "trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="], + "ts-interface-checker": ["ts-interface-checker@0.1.13", "", {}, "sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA=="], + "tsconfck": ["tsconfck@3.1.5", "", { "peerDependencies": { "typescript": "^5.0.0" }, "optionalPeers": ["typescript"], "bin": { "tsconfck": "bin/tsconfck.js" } }, "sha512-CLDfGgUp7XPswWnezWwsCRxNmgQjhYq3VXHM0/XIRxhVrKw0M1if9agzryh1QS3nxjCROvV+xWxoJO1YctzzWg=="], - "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "tsconfig-paths": ["tsconfig-paths@3.15.0", "", { "dependencies": { "@types/json5": "^0.0.29", "json5": "^1.0.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" } }, "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg=="], + + "tslib": ["tslib@1.14.1", "", {}, "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg=="], "type-fest": ["type-fest@2.19.0", "", {}, "sha512-RAH822pAdBgcNMAfWnCBU3CFZcfZ/i1eZjwFU/dsLKumyuuP3niueg2UAukXYF0E2AAoc82ZSSf9J0WQBinzHA=="], @@ -1109,18 +1238,28 @@ "vite": ["vite@6.3.3", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.4.4", "picomatch": "^4.0.2", "postcss": "^8.5.3", "rollup": "^4.34.9", "tinyglobby": "^0.2.13" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "jiti": ">=1.21.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-5nXH+QsELbFKhsEfWLkHrvgRpTdGJzqOZ+utSdmPTvwHmvU6ITTm3xx+mRusihkcI8GeC7lCDyn3kDtiki9scw=="], + "vite-node": ["vite-node@3.1.2", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.4.0", "es-module-lexer": "^1.6.0", "pathe": "^2.0.3", "vite": "^5.0.0 || ^6.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-/8iMryv46J3aK13iUXsei5G/A3CUlW4665THCPS+K8xAaqrVWiGB4RfXMQXCLjpK9P2eK//BczrVkn5JLAk6DA=="], + "vite-tsconfig-paths": ["vite-tsconfig-paths@5.1.4", "", { "dependencies": { "debug": "^4.1.1", "globrex": "^0.1.2", "tsconfck": "^3.0.3" }, "peerDependencies": { "vite": "*" }, "optionalPeers": ["vite"] }, "sha512-cYj0LRuLV2c2sMqhqhGpaO3LretdtMn/BVX4cPLanIZuwwrkVl+lK84E/miEXkCHWXuq65rhNN4rXsBcOB3S4w=="], "vitefu": ["vitefu@1.0.6", "", { "peerDependencies": { "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0" }, "optionalPeers": ["vite"] }, "sha512-+Rex1GlappUyNN6UfwbVZne/9cYC4+R2XDk9xkNXBKMw6HQagdX9PgZ8V2v1WUSK1wfBLp7qbI1+XSNIlB1xmA=="], + "vitest": ["vitest@3.1.2", "", { "dependencies": { "@vitest/expect": "3.1.2", "@vitest/mocker": "3.1.2", "@vitest/pretty-format": "^3.1.2", "@vitest/runner": "3.1.2", "@vitest/snapshot": "3.1.2", "@vitest/spy": "3.1.2", "@vitest/utils": "3.1.2", "chai": "^5.2.0", "debug": "^4.4.0", "expect-type": "^1.2.1", "magic-string": "^0.30.17", "pathe": "^2.0.3", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.13", "tinypool": "^1.0.2", "tinyrainbow": "^2.0.0", "vite": "^5.0.0 || ^6.0.0", "vite-node": "3.1.2", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", "@vitest/browser": "3.1.2", "@vitest/ui": "3.1.2", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-WaxpJe092ID1C0mr+LH9MmNrhfzi8I65EX/NRU/Ld016KqQNRgxSOlGNP1hHN+a/F8L15Mh8klwaF77zR3GeDQ=="], + + "vitest-tsconfig-paths": ["vitest-tsconfig-paths@3.4.1", "", { "dependencies": { "debug": "^4.1.1", "globrex": "^0.1.2", "recrawl-sync": "^2.0.3", "tsconfig-paths": "^3.9.0" } }, "sha512-CnRpA/jcqgZfnkk0yvwFW92UmIpf03wX/wLiQBNWAcOG7nv6Sdz3GsPESAMEqbVy8kHBoWB3XeNamu6PUrFZLA=="], + "web-namespaces": ["web-namespaces@2.0.1", "", {}, "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ=="], "webidl-conversions": ["webidl-conversions@3.0.1", "", {}, "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="], "whatwg-url": ["whatwg-url@5.0.0", "", { "dependencies": { "tr46": "~0.0.3", "webidl-conversions": "^3.0.0" } }, "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw=="], + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + "which-pm-runs": ["which-pm-runs@1.1.0", "", {}, "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA=="], + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "widest-line": ["widest-line@5.0.0", "", { "dependencies": { "string-width": "^7.0.0" } }, "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA=="], "winston": ["winston@3.17.0", "", { "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.2", "async": "^3.2.3", "is-stream": "^2.0.0", "logform": "^2.7.0", "one-time": "^1.0.0", "readable-stream": "^3.4.0", "safe-stable-stringify": "^2.3.1", "stack-trace": "0.0.x", "triple-beam": "^1.3.0", "winston-transport": "^4.9.0" } }, "sha512-DLiFIXYC5fMPxaRg832S6F5mJYvePtmO5G9v9IgUFPhXm9/GkXarH/TUrBAVzhTCzAj9anE/+GjrgXp/54nOgw=="], @@ -1129,6 +1268,8 @@ "wrap-ansi": ["wrap-ansi@9.0.0", "", { "dependencies": { "ansi-styles": "^6.2.1", "string-width": "^7.0.0", "strip-ansi": "^7.1.0" } }, "sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q=="], + "wrap-ansi-cjs": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], @@ -1155,8 +1296,16 @@ "zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="], + "@emnapi/runtime/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "@isaacs/cliui/string-width": ["string-width@5.1.2", "", { "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", "strip-ansi": "^7.0.1" } }, "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA=="], + + "@isaacs/cliui/wrap-ansi": ["wrap-ansi@8.1.0", "", { "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", "strip-ansi": "^7.0.1" } }, "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ=="], + "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="], + "@swc/helpers/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "@tailwindcss/oxide-wasm32-wasi/@emnapi/core": ["@emnapi/core@1.4.3", "", { "dependencies": { "@emnapi/wasi-threads": "1.0.2", "tslib": "^2.4.0" }, "bundled": true }, "sha512-4m62DuCE07lw01soJwPiBGC0nAww0Q+RY70VZ+n49yDIO13yyinhbWCeNnaob0lakDtWQzSdtNWzJeOJt2ma+g=="], "@tailwindcss/oxide-wasm32-wasi/@emnapi/runtime": ["@emnapi/runtime@1.4.3", "", { "dependencies": { "tslib": "^2.4.0" }, "bundled": true }, "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ=="], @@ -1183,10 +1332,30 @@ "hast-util-to-parse5/property-information": ["property-information@6.5.0", "", {}, "sha512-PgTgs/BlvHxOu8QuEN7wi5A0OmXaBcHpmCSTehcs6Uuu9IkDIEo13Hy7n898RHfrQ49vKCoGeWZSaAK01nwVig=="], + "lower-case/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "no-case/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "pascal-case/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "pg/pg-types": ["pg-types@2.2.0", "", { "dependencies": { "pg-int8": "1.0.1", "postgres-array": "~2.0.0", "postgres-bytea": "~1.0.0", "postgres-date": "~1.0.4", "postgres-interval": "^1.1.0" } }, "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA=="], "prompts/kleur": ["kleur@3.0.3", "", {}, "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w=="], + "string-width-cjs/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "string-width-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "strip-ansi-cjs/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "wrap-ansi-cjs/ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "wrap-ansi-cjs/string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "wrap-ansi-cjs/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "@isaacs/cliui/string-width/emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], + "ansi-align/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], "ansi-align/string-width/strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], @@ -1203,6 +1372,12 @@ "pg/pg-types/postgres-interval": ["postgres-interval@1.2.0", "", { "dependencies": { "xtend": "^4.0.0" } }, "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ=="], + "string-width-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "wrap-ansi-cjs/string-width/emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "wrap-ansi-cjs/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + "ansi-align/string-width/strip-ansi/ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "colorspace/color/color-convert/color-name": ["color-name@1.1.3", "", {}, "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw=="], diff --git a/package.json b/package.json index d98fa16..2afb5de 100644 --- a/package.json +++ b/package.json @@ -6,9 +6,15 @@ "workspaces": [ "packages/frontend", "packages/core", - "packages/backend" + "packages/backend", + "packages/crawler" ], "dependencies": { "postgres": "^3.4.5" + }, + "devDependencies": { + "vite-tsconfig-paths": "^5.1.4", + "vitest": "^3.1.2", + "vitest-tsconfig-paths": "^3.4.1" } } diff --git a/packages/core/db/dbNew.ts b/packages/core/db/dbNew.ts index 187a7a6..11088e5 100644 --- a/packages/core/db/dbNew.ts +++ b/packages/core/db/dbNew.ts @@ -2,3 +2,5 @@ import postgres from "postgres"; import { postgresConfigNpm } from "./pgConfigNew"; export const sql = postgres(postgresConfigNpm); + +export const sqlTest = postgres(postgresConfigNpm); \ No newline at end of file diff --git a/packages/core/db/schema.d.ts b/packages/core/db/schema.d.ts index a9a7296..f10c8bc 100644 --- a/packages/core/db/schema.d.ts +++ b/packages/core/db/schema.d.ts @@ -1,6 +1,6 @@ export interface AllDataType { id: number; - aid: bigint; + aid: number; bvid: string | null; description: string | null; uid: number | null; @@ -28,12 +28,12 @@ export interface VideoSnapshotType { favorites: number; shares: number; danmakus: number; - aid: bigint; + aid: number; replies: number; } export interface LatestSnapshotType { - aid: bigint; + aid: number; time: number; views: number; danmakus: number; @@ -46,7 +46,7 @@ export interface LatestSnapshotType { export interface SnapshotScheduleType { id: number; - aid: bigint; + aid: number; type?: string; created_at: string; started_at?: string; diff --git a/packages/core/log/test.ts b/packages/core/log/test.ts index ee5953c..0c48452 100644 --- a/packages/core/log/test.ts +++ b/packages/core/log/test.ts @@ -1,4 +1,4 @@ -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; logger.error(Error("test error"), "test service"); logger.debug(`some string`); diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index 519f871..e7ca3f1 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -1,4 +1,4 @@ -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts"; import { SlidingWindow } from "mq/slidingWindow.ts"; import { redis } from "db/redis.ts"; diff --git a/packages/crawler/.gitignore b/packages/crawler/.gitignore new file mode 100644 index 0000000..8e12416 --- /dev/null +++ b/packages/crawler/.gitignore @@ -0,0 +1,2 @@ +.cache +build/ \ No newline at end of file diff --git a/packages/crawler/bun.lock b/packages/crawler/bun.lock new file mode 100644 index 0000000..e7559ee --- /dev/null +++ b/packages/crawler/bun.lock @@ -0,0 +1,433 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "core", + "dependencies": { + "@bull-board/api": "^6.9.5", + "@bull-board/express": "^6.9.5", + "@huggingface/transformers": "^3.5.1", + "bullmq": "^5.52.1", + "express": "^5.1.0", + "ioredis": "^5.6.1", + "onnxruntime-node": "1.19.2", + }, + "devDependencies": { + "concurrently": "^9.1.2", + }, + }, + }, + "packages": { + "@bull-board/api": ["@bull-board/api@6.9.5", "", { "dependencies": { "redis-info": "^3.1.0" }, "peerDependencies": { "@bull-board/ui": "6.9.5" } }, "sha512-9d03Mu9fuQ3sHAxAwUmlTo8C9KwKHd6Ef2IUOZMG93cAHsQ0oaOpYc5d0roZacYFvfYKr7bTANptKO1f3GLp2g=="], + + "@bull-board/express": ["@bull-board/express@6.9.5", "", { "dependencies": { "@bull-board/api": "6.9.5", "@bull-board/ui": "6.9.5", "ejs": "^3.1.10", "express": "^4.21.1 || ^5.0.0" } }, "sha512-N2B/RSEX/EpIvT/uOJXCACevElv2CUEqrvgoLuwM2n1Fb3VWaatj/S004VLUtHjOCPfODosJ1xoxelelQcV3Dw=="], + + "@bull-board/ui": ["@bull-board/ui@6.9.5", "", { "dependencies": { "@bull-board/api": "6.9.5" } }, "sha512-+4YnDvyuY3MOVkXFxkspRbqawLtIExHzRQ4raQWagOc35KD7v2/ccFGyRPDI/N0bemsiNkOPkcZGf/LFcaOZmA=="], + + "@emnapi/runtime": ["@emnapi/runtime@1.4.3", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-pBPWdu6MLKROBX05wSNKcNb++m5Er+KQ9QkB+WVM+pW2Kx9hoSrVTnu3BdkI5eBLZoKu/J6mW/B6i6bJB2ytXQ=="], + + "@huggingface/jinja": ["@huggingface/jinja@0.4.1", "", {}, "sha512-3WXbMFaPkk03LRCM0z0sylmn8ddDm4ubjU7X+Hg4M2GOuMklwoGAFXp9V2keq7vltoB/c7McE5aHUVVddAewsw=="], + + "@huggingface/transformers": ["@huggingface/transformers@3.5.1", "", { "dependencies": { "@huggingface/jinja": "^0.4.1", "onnxruntime-node": "1.21.0", "onnxruntime-web": "1.22.0-dev.20250409-89f8206ba4", "sharp": "^0.34.1" } }, "sha512-qWsPoJMBPYcrGuzRMVL//3dwcLXED9r+j+Hzw+hZpBfrMPdyq57ehNs7lew0D34Paj0DO0E0kZQjOZcIVN17hQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.1.0" }, "os": "darwin", "cpu": "arm64" }, "sha512-pn44xgBtgpEbZsu+lWf2KNb6OAf70X68k+yk69Ic2Xz11zHR/w24/U49XT7AeRwJ0Px+mhALhU5LPci1Aymk7A=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.1.0" }, "os": "darwin", "cpu": "x64" }, "sha512-VfuYgG2r8BpYiOUN+BfYeFo69nP/MIwAtSJ7/Zpxc5QF3KS22z8Pvg3FkrSFJBPNQ7mmcUcYQFBmEQp7eu1F8Q=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.1.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-HZ/JUmPwrJSoM4DIQPv/BfNh9yrOA8tlBbqbLz4JZ5uew2+o22Ik+tHQJcih7QJuSa0zo5coHTfD5J8inqj9DA=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.1.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-Xzc2ToEmHN+hfvsl9wja0RlnXEgpKNmftriQp6XzY/RaSfwD9th+MSh0WQKzUreLKKINb3afirxW7A0fz2YWuQ=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.1.0", "", { "os": "linux", "cpu": "arm" }, "sha512-s8BAd0lwUIvYCJyRdFqvsj+BJIpDBSxs6ivrOPm/R7piTs5UIwY5OjXrP2bqXC9/moGsyRa37eYWYCOGVXxVrA=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-IVfGJa7gjChDET1dK9SekxFFdflarnUB8PwW8aGwEoF3oAsSDuNUTYS+SKDOyOJxQyDC1aPFMuRYLoDInyV9Ew=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.1.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-tiXxFZFbhnkWE2LA8oQj7KYR+bWBkiV2nilRldT7bqoEZ4HiDOcePr9wVDAZPi/Id5fT1oY9iGnDq20cwUz8lQ=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.1.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-xukSwvhguw7COyzvmjydRb3x/09+21HykyapcZchiCUkTThEQEOMtBj9UhkaBRLuBrgLFzQ2wbxdeCCJW/jgJA=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-yRj2+reB8iMg9W5sULM3S74jVS7zqSzHG3Ol/twnAAkAhnGQnpjj6e4ayUz7V+FpKypwgs82xbRdYtchTTUB+Q=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.1.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-jYZdG+whg0MDK+q2COKbYidaqW/WTz0cc1E+tMAusiDygrM4ypmSCjOJPmFTvHHJ8j/6cAGyeDWZOsK06tP33w=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.1.0", "", { "os": "linux", "cpu": "x64" }, "sha512-wK7SBdwrAiycjXdkPnGCPLjYb9lD4l6Ze2gSdAGVZrEL05AOUJESWU2lhlC+Ffn5/G+VKuSm6zzbQSzFX/P65A=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.1.0" }, "os": "linux", "cpu": "arm" }, "sha512-anKiszvACti2sGy9CirTlNyk7BjjZPiML1jt2ZkTdcvpLU1YH6CXwRAZCA2UmRXnhiIftXQ7+Oh62Ji25W72jA=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.1.0" }, "os": "linux", "cpu": "arm64" }, "sha512-kX2c+vbvaXC6vly1RDf/IWNXxrlxLNpBVWkdpRq5Ka7OOKj6nr66etKy2IENf6FtOgklkg9ZdGpEu9kwdlcwOQ=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.1.0" }, "os": "linux", "cpu": "s390x" }, "sha512-7s0KX2tI9mZI2buRipKIw2X1ufdTeaRgwmRabt5bi9chYfhur+/C1OXg3TKg/eag1W+6CCWLVmSauV1owmRPxA=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.1.0" }, "os": "linux", "cpu": "x64" }, "sha512-wExv7SH9nmoBW3Wr2gvQopX1k8q2g5V5Iag8Zk6AVENsjwd+3adjwxtp3Dcu2QhOXr8W9NusBU6XcQUohBZ5MA=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.1.0" }, "os": "linux", "cpu": "arm64" }, "sha512-DfvyxzHxw4WGdPiTF0SOHnm11Xv4aQexvqhRDAoD00MzHekAj9a/jADXeXYCDFH/DzYruwHbXU7uz+H+nWmSOQ=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.1", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.1.0" }, "os": "linux", "cpu": "x64" }, "sha512-pax/kTR407vNb9qaSIiWVnQplPcGU8LRIJpDT5o8PdAx5aAA7AS3X9PS8Isw1/WfqgQorPotjrZL3Pqh6C5EBg=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.1", "", { "dependencies": { "@emnapi/runtime": "^1.4.0" }, "cpu": "none" }, "sha512-YDybQnYrLQfEpzGOQe7OKcyLUCML4YOXl428gOOzBgN6Gw0rv8dpsJ7PqTHxBnXnwXr8S1mYFSLSa727tpz0xg=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.1", "", { "os": "win32", "cpu": "ia32" }, "sha512-WKf/NAZITnonBf3U1LfdjoMgNO5JYRSlhovhRhMxXVdvWYveM4kM3L8m35onYIdh75cOMCo1BexgVQcCDzyoWw=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.1", "", { "os": "win32", "cpu": "x64" }, "sha512-hw1iIAHpNE8q3uMIRCgGOeDoz9KtFNarFLQclLxr/LK1VBkj8nby18RjFvr6aP7USRYAjTZW6yisnBWMX571Tw=="], + + "@ioredis/commands": ["@ioredis/commands@1.2.0", "", {}, "sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg=="], + + "@isaacs/fs-minipass": ["@isaacs/fs-minipass@4.0.1", "", { "dependencies": { "minipass": "^7.0.4" } }, "sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w=="], + + "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], + + "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm": ["@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3", "", { "os": "linux", "cpu": "arm" }, "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw=="], + + "@msgpackr-extract/msgpackr-extract-linux-arm64": ["@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3", "", { "os": "linux", "cpu": "arm64" }, "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg=="], + + "@msgpackr-extract/msgpackr-extract-linux-x64": ["@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3", "", { "os": "linux", "cpu": "x64" }, "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg=="], + + "@msgpackr-extract/msgpackr-extract-win32-x64": ["@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3", "", { "os": "win32", "cpu": "x64" }, "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ=="], + + "@protobufjs/aspromise": ["@protobufjs/aspromise@1.1.2", "", {}, "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ=="], + + "@protobufjs/base64": ["@protobufjs/base64@1.1.2", "", {}, "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg=="], + + "@protobufjs/codegen": ["@protobufjs/codegen@2.0.4", "", {}, "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg=="], + + "@protobufjs/eventemitter": ["@protobufjs/eventemitter@1.1.0", "", {}, "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q=="], + + "@protobufjs/fetch": ["@protobufjs/fetch@1.1.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.1", "@protobufjs/inquire": "^1.1.0" } }, "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ=="], + + "@protobufjs/float": ["@protobufjs/float@1.0.2", "", {}, "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ=="], + + "@protobufjs/inquire": ["@protobufjs/inquire@1.1.0", "", {}, "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q=="], + + "@protobufjs/path": ["@protobufjs/path@1.1.2", "", {}, "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA=="], + + "@protobufjs/pool": ["@protobufjs/pool@1.1.0", "", {}, "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw=="], + + "@protobufjs/utf8": ["@protobufjs/utf8@1.1.0", "", {}, "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw=="], + + "@types/node": ["@types/node@22.15.3", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-lX7HFZeHf4QG/J7tBZqrCAXwz9J5RD56Y6MpP0eJkka8p+K0RY/yBTW7CYFJ4VGCclxqOLKmiGP5juQc6MKgcw=="], + + "accepts": ["accepts@2.0.0", "", { "dependencies": { "mime-types": "^3.0.0", "negotiator": "^1.0.0" } }, "sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng=="], + + "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], + + "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + + "async": ["async@3.2.6", "", {}, "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA=="], + + "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], + + "body-parser": ["body-parser@2.2.0", "", { "dependencies": { "bytes": "^3.1.2", "content-type": "^1.0.5", "debug": "^4.4.0", "http-errors": "^2.0.0", "iconv-lite": "^0.6.3", "on-finished": "^2.4.1", "qs": "^6.14.0", "raw-body": "^3.0.0", "type-is": "^2.0.0" } }, "sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg=="], + + "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], + + "brace-expansion": ["brace-expansion@1.1.11", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA=="], + + "bullmq": ["bullmq@5.52.1", "", { "dependencies": { "cron-parser": "^4.9.0", "ioredis": "^5.4.1", "msgpackr": "^1.11.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-u7CSV9wID3MBEX2DNubEErbAlrADgm8abUBAi6h8rQTnuTkhhgMs2iD7uhqplK8lIgUOkBIW3sDJWaMSInH47A=="], + + "bytes": ["bytes@3.1.2", "", {}, "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg=="], + + "call-bind-apply-helpers": ["call-bind-apply-helpers@1.0.2", "", { "dependencies": { "es-errors": "^1.3.0", "function-bind": "^1.1.2" } }, "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ=="], + + "call-bound": ["call-bound@1.0.4", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "get-intrinsic": "^1.3.0" } }, "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg=="], + + "chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="], + + "chownr": ["chownr@3.0.0", "", {}, "sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g=="], + + "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], + + "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], + + "color": ["color@4.2.3", "", { "dependencies": { "color-convert": "^2.0.1", "color-string": "^1.9.0" } }, "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A=="], + + "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], + + "color-name": ["color-name@1.1.4", "", {}, "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="], + + "color-string": ["color-string@1.9.1", "", { "dependencies": { "color-name": "^1.0.0", "simple-swizzle": "^0.2.2" } }, "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg=="], + + "concat-map": ["concat-map@0.0.1", "", {}, "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg=="], + + "concurrently": ["concurrently@9.1.2", "", { "dependencies": { "chalk": "^4.1.2", "lodash": "^4.17.21", "rxjs": "^7.8.1", "shell-quote": "^1.8.1", "supports-color": "^8.1.1", "tree-kill": "^1.2.2", "yargs": "^17.7.2" }, "bin": { "concurrently": "dist/bin/concurrently.js", "conc": "dist/bin/concurrently.js" } }, "sha512-H9MWcoPsYddwbOGM6difjVwVZHl63nwMEwDJG/L7VGtuaJhb12h2caPG2tVPWs7emuYix252iGfqOyrz1GczTQ=="], + + "content-disposition": ["content-disposition@1.0.0", "", { "dependencies": { "safe-buffer": "5.2.1" } }, "sha512-Au9nRL8VNUut/XSzbQA38+M78dzP4D+eqg3gfJHMIHHYa3bg067xj1KxMUWj+VULbiZMowKngFFbKczUrNJ1mg=="], + + "content-type": ["content-type@1.0.5", "", {}, "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA=="], + + "cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + + "cookie-signature": ["cookie-signature@1.2.2", "", {}, "sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg=="], + + "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + + "debug": ["debug@4.4.0", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA=="], + + "define-data-property": ["define-data-property@1.1.4", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="], + + "define-properties": ["define-properties@1.2.1", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="], + + "denque": ["denque@2.1.0", "", {}, "sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw=="], + + "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + + "detect-libc": ["detect-libc@2.0.4", "", {}, "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA=="], + + "detect-node": ["detect-node@2.1.0", "", {}, "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g=="], + + "dunder-proto": ["dunder-proto@1.0.1", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.1", "es-errors": "^1.3.0", "gopd": "^1.2.0" } }, "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A=="], + + "ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="], + + "ejs": ["ejs@3.1.10", "", { "dependencies": { "jake": "^10.8.5" }, "bin": { "ejs": "bin/cli.js" } }, "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA=="], + + "emoji-regex": ["emoji-regex@8.0.0", "", {}, "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="], + + "encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="], + + "es-define-property": ["es-define-property@1.0.1", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="], + + "es-errors": ["es-errors@1.3.0", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="], + + "es-object-atoms": ["es-object-atoms@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0" } }, "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA=="], + + "es6-error": ["es6-error@4.1.1", "", {}, "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg=="], + + "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], + + "escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="], + + "escape-string-regexp": ["escape-string-regexp@4.0.0", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="], + + "etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="], + + "express": ["express@5.1.0", "", { "dependencies": { "accepts": "^2.0.0", "body-parser": "^2.2.0", "content-disposition": "^1.0.0", "content-type": "^1.0.5", "cookie": "^0.7.1", "cookie-signature": "^1.2.1", "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "finalhandler": "^2.1.0", "fresh": "^2.0.0", "http-errors": "^2.0.0", "merge-descriptors": "^2.0.0", "mime-types": "^3.0.0", "on-finished": "^2.4.1", "once": "^1.4.0", "parseurl": "^1.3.3", "proxy-addr": "^2.0.7", "qs": "^6.14.0", "range-parser": "^1.2.1", "router": "^2.2.0", "send": "^1.1.0", "serve-static": "^2.2.0", "statuses": "^2.0.1", "type-is": "^2.0.1", "vary": "^1.1.2" } }, "sha512-DT9ck5YIRU+8GYzzU5kT3eHGA5iL+1Zd0EutOmTE9Dtk+Tvuzd23VBU+ec7HPNSTxXYO55gPV/hq4pSBJDjFpA=="], + + "filelist": ["filelist@1.0.4", "", { "dependencies": { "minimatch": "^5.0.1" } }, "sha512-w1cEuf3S+DrLCQL7ET6kz+gmlJdbq9J7yXCSjK/OZCPA+qEN1WyF4ZAf0YYJa4/shHJra2t/d/r8SV4Ji+x+8Q=="], + + "finalhandler": ["finalhandler@2.1.0", "", { "dependencies": { "debug": "^4.4.0", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "on-finished": "^2.4.1", "parseurl": "^1.3.3", "statuses": "^2.0.1" } }, "sha512-/t88Ty3d5JWQbWYgaOGCCYfXRwV1+be02WqYYlL6h0lEiUAMPM8o8qKGO01YIkOHzka2up08wvgYD0mDiI+q3Q=="], + + "flatbuffers": ["flatbuffers@25.2.10", "", {}, "sha512-7JlN9ZvLDG1McO3kbX0k4v+SUAg48L1rIwEvN6ZQl/eCtgJz9UylTMzE9wrmYrcorgxm3CX/3T/w5VAub99UUw=="], + + "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], + + "fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="], + + "function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="], + + "get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="], + + "get-intrinsic": ["get-intrinsic@1.3.0", "", { "dependencies": { "call-bind-apply-helpers": "^1.0.2", "es-define-property": "^1.0.1", "es-errors": "^1.3.0", "es-object-atoms": "^1.1.1", "function-bind": "^1.1.2", "get-proto": "^1.0.1", "gopd": "^1.2.0", "has-symbols": "^1.1.0", "hasown": "^2.0.2", "math-intrinsics": "^1.1.0" } }, "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ=="], + + "get-proto": ["get-proto@1.0.1", "", { "dependencies": { "dunder-proto": "^1.0.1", "es-object-atoms": "^1.0.0" } }, "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g=="], + + "global-agent": ["global-agent@3.0.0", "", { "dependencies": { "boolean": "^3.0.1", "es6-error": "^4.1.1", "matcher": "^3.0.0", "roarr": "^2.15.3", "semver": "^7.3.2", "serialize-error": "^7.0.1" } }, "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q=="], + + "globalthis": ["globalthis@1.0.4", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="], + + "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], + + "guid-typescript": ["guid-typescript@1.0.9", "", {}, "sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ=="], + + "has-flag": ["has-flag@4.0.0", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="], + + "has-property-descriptors": ["has-property-descriptors@1.0.2", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="], + + "has-symbols": ["has-symbols@1.1.0", "", {}, "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ=="], + + "hasown": ["hasown@2.0.2", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="], + + "http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="], + + "iconv-lite": ["iconv-lite@0.6.3", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="], + + "inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="], + + "ioredis": ["ioredis@5.6.1", "", { "dependencies": { "@ioredis/commands": "^1.1.1", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-UxC0Yv1Y4WRJiGQxQkP0hfdL0/5/6YvdfOOClRgJ0qppSarkhneSa6UvkMkms0AkdGimSH3Ikqm+6mkMmX7vGA=="], + + "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], + + "is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="], + + "is-fullwidth-code-point": ["is-fullwidth-code-point@3.0.0", "", {}, "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="], + + "is-promise": ["is-promise@4.0.0", "", {}, "sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ=="], + + "jake": ["jake@10.9.2", "", { "dependencies": { "async": "^3.2.3", "chalk": "^4.0.2", "filelist": "^1.0.4", "minimatch": "^3.1.2" }, "bin": { "jake": "bin/cli.js" } }, "sha512-2P4SQ0HrLQ+fw6llpLnOaGAvN2Zu6778SJMrCUwns4fOoG9ayrTiZk3VV8sCPkVZF8ab0zksVpS8FDY5pRCNBA=="], + + "json-stringify-safe": ["json-stringify-safe@5.0.1", "", {}, "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA=="], + + "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], + + "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], + + "lodash.isarguments": ["lodash.isarguments@3.1.0", "", {}, "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg=="], + + "long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="], + + "luxon": ["luxon@3.6.1", "", {}, "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ=="], + + "matcher": ["matcher@3.0.0", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng=="], + + "math-intrinsics": ["math-intrinsics@1.1.0", "", {}, "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="], + + "media-typer": ["media-typer@1.1.0", "", {}, "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw=="], + + "merge-descriptors": ["merge-descriptors@2.0.0", "", {}, "sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g=="], + + "mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="], + + "mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="], + + "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + + "minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="], + + "minizlib": ["minizlib@3.0.2", "", { "dependencies": { "minipass": "^7.1.2" } }, "sha512-oG62iEk+CYt5Xj2YqI5Xi9xWUeZhDI8jjQmC5oThVH5JGCTgIjr7ciJDzC7MBzYd//WvR1OTmP5Q38Q8ShQtVA=="], + + "mkdirp": ["mkdirp@3.0.1", "", { "bin": { "mkdirp": "dist/cjs/src/bin.js" } }, "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "msgpackr": ["msgpackr@1.11.2", "", { "optionalDependencies": { "msgpackr-extract": "^3.0.2" } }, "sha512-F9UngXRlPyWCDEASDpTf6c9uNhGPTqnTeLVt7bN+bU1eajoR/8V9ys2BRaV5C/e5ihE6sJ9uPIKaYt6bFuO32g=="], + + "msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], + + "negotiator": ["negotiator@1.0.0", "", {}, "sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg=="], + + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], + + "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], + + "object-inspect": ["object-inspect@1.13.4", "", {}, "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew=="], + + "object-keys": ["object-keys@1.1.1", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="], + + "on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="], + + "once": ["once@1.4.0", "", { "dependencies": { "wrappy": "1" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="], + + "onnxruntime-common": ["onnxruntime-common@1.19.2", "", {}, "sha512-a4R7wYEVFbZBlp0BfhpbFWqe4opCor3KM+5Wm22Az3NGDcQMiU2hfG/0MfnBs+1ZrlSGmlgWeMcXQkDk1UFb8Q=="], + + "onnxruntime-node": ["onnxruntime-node@1.19.2", "", { "dependencies": { "onnxruntime-common": "1.19.2", "tar": "^7.0.1" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-9eHMP/HKbbeUcqte1JYzaaRC8JPn7ojWeCeoyShO86TOR97OCyIyAIOGX3V95ErjslVhJRXY8Em/caIUc0hm1Q=="], + + "onnxruntime-web": ["onnxruntime-web@1.22.0-dev.20250409-89f8206ba4", "", { "dependencies": { "flatbuffers": "^25.1.24", "guid-typescript": "^1.0.9", "long": "^5.2.3", "onnxruntime-common": "1.22.0-dev.20250409-89f8206ba4", "platform": "^1.3.6", "protobufjs": "^7.2.4" } }, "sha512-0uS76OPgH0hWCPrFKlL8kYVV7ckM7t/36HfbgoFw6Nd0CZVVbQC4PkrR8mBX8LtNUFZO25IQBqV2Hx2ho3FlbQ=="], + + "parseurl": ["parseurl@1.3.3", "", {}, "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ=="], + + "path-to-regexp": ["path-to-regexp@8.2.0", "", {}, "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ=="], + + "platform": ["platform@1.3.6", "", {}, "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg=="], + + "protobufjs": ["protobufjs@7.5.0", "", { "dependencies": { "@protobufjs/aspromise": "^1.1.2", "@protobufjs/base64": "^1.1.2", "@protobufjs/codegen": "^2.0.4", "@protobufjs/eventemitter": "^1.1.0", "@protobufjs/fetch": "^1.1.0", "@protobufjs/float": "^1.0.2", "@protobufjs/inquire": "^1.1.0", "@protobufjs/path": "^1.1.2", "@protobufjs/pool": "^1.1.0", "@protobufjs/utf8": "^1.1.0", "@types/node": ">=13.7.0", "long": "^5.0.0" } }, "sha512-Z2E/kOY1QjoMlCytmexzYfDm/w5fKAiRwpSzGtdnXW1zC88Z2yXazHHrOtwCzn+7wSxyE8PYM4rvVcMphF9sOA=="], + + "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], + + "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], + + "range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="], + + "raw-body": ["raw-body@3.0.0", "", { "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", "iconv-lite": "0.6.3", "unpipe": "1.0.0" } }, "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g=="], + + "redis-errors": ["redis-errors@1.2.0", "", {}, "sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w=="], + + "redis-info": ["redis-info@3.1.0", "", { "dependencies": { "lodash": "^4.17.11" } }, "sha512-ER4L9Sh/vm63DkIE0bkSjxluQlioBiBgf5w1UuldaW/3vPcecdljVDisZhmnCMvsxHNiARTTDDHGg9cGwTfrKg=="], + + "redis-parser": ["redis-parser@3.0.0", "", { "dependencies": { "redis-errors": "^1.0.0" } }, "sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A=="], + + "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], + + "roarr": ["roarr@2.15.4", "", { "dependencies": { "boolean": "^3.0.1", "detect-node": "^2.0.4", "globalthis": "^1.0.1", "json-stringify-safe": "^5.0.1", "semver-compare": "^1.0.0", "sprintf-js": "^1.1.2" } }, "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A=="], + + "router": ["router@2.2.0", "", { "dependencies": { "debug": "^4.4.0", "depd": "^2.0.0", "is-promise": "^4.0.0", "parseurl": "^1.3.3", "path-to-regexp": "^8.0.0" } }, "sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ=="], + + "rxjs": ["rxjs@7.8.2", "", { "dependencies": { "tslib": "^2.1.0" } }, "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA=="], + + "safe-buffer": ["safe-buffer@5.2.1", "", {}, "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="], + + "safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="], + + "semver": ["semver@7.7.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA=="], + + "semver-compare": ["semver-compare@1.0.0", "", {}, "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow=="], + + "send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="], + + "serialize-error": ["serialize-error@7.0.1", "", { "dependencies": { "type-fest": "^0.13.1" } }, "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw=="], + + "serve-static": ["serve-static@2.2.0", "", { "dependencies": { "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "parseurl": "^1.3.3", "send": "^1.2.0" } }, "sha512-61g9pCh0Vnh7IutZjtLGGpTA355+OPn2TyDv/6ivP2h/AdAVX9azsoxmg2/M6nZeQZNYBEwIcsne1mJd9oQItQ=="], + + "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + + "sharp": ["sharp@0.34.1", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.7.1" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.1", "@img/sharp-darwin-x64": "0.34.1", "@img/sharp-libvips-darwin-arm64": "1.1.0", "@img/sharp-libvips-darwin-x64": "1.1.0", "@img/sharp-libvips-linux-arm": "1.1.0", "@img/sharp-libvips-linux-arm64": "1.1.0", "@img/sharp-libvips-linux-ppc64": "1.1.0", "@img/sharp-libvips-linux-s390x": "1.1.0", "@img/sharp-libvips-linux-x64": "1.1.0", "@img/sharp-libvips-linuxmusl-arm64": "1.1.0", "@img/sharp-libvips-linuxmusl-x64": "1.1.0", "@img/sharp-linux-arm": "0.34.1", "@img/sharp-linux-arm64": "0.34.1", "@img/sharp-linux-s390x": "0.34.1", "@img/sharp-linux-x64": "0.34.1", "@img/sharp-linuxmusl-arm64": "0.34.1", "@img/sharp-linuxmusl-x64": "0.34.1", "@img/sharp-wasm32": "0.34.1", "@img/sharp-win32-ia32": "0.34.1", "@img/sharp-win32-x64": "0.34.1" } }, "sha512-1j0w61+eVxu7DawFJtnfYcvSv6qPFvfTaqzTQ2BLknVhHTwGS8sc63ZBF4rzkWMBVKybo4S5OBtDdZahh2A1xg=="], + + "shell-quote": ["shell-quote@1.8.2", "", {}, "sha512-AzqKpGKjrj7EM6rKVQEPpB288oCfnrEIuyoT9cyF4nmGa7V8Zk6f7RRqYisX8X9m+Q7bd632aZW4ky7EhbQztA=="], + + "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + + "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + + "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], + + "side-channel-weakmap": ["side-channel-weakmap@1.0.2", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3", "side-channel-map": "^1.0.1" } }, "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A=="], + + "simple-swizzle": ["simple-swizzle@0.2.2", "", { "dependencies": { "is-arrayish": "^0.3.1" } }, "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg=="], + + "sprintf-js": ["sprintf-js@1.1.3", "", {}, "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA=="], + + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + + "statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="], + + "string-width": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="], + + "strip-ansi": ["strip-ansi@6.0.1", "", { "dependencies": { "ansi-regex": "^5.0.1" } }, "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A=="], + + "supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="], + + "tar": ["tar@7.4.3", "", { "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", "minizlib": "^3.0.1", "mkdirp": "^3.0.1", "yallist": "^5.0.0" } }, "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw=="], + + "toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="], + + "tree-kill": ["tree-kill@1.2.2", "", { "bin": { "tree-kill": "cli.js" } }, "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A=="], + + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + + "type-fest": ["type-fest@0.13.1", "", {}, "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg=="], + + "type-is": ["type-is@2.0.1", "", { "dependencies": { "content-type": "^1.0.5", "media-typer": "^1.1.0", "mime-types": "^3.0.0" } }, "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "unpipe": ["unpipe@1.0.0", "", {}, "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ=="], + + "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], + + "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], + + "wrap-ansi": ["wrap-ansi@7.0.0", "", { "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", "strip-ansi": "^6.0.0" } }, "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q=="], + + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + + "y18n": ["y18n@5.0.8", "", {}, "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA=="], + + "yallist": ["yallist@5.0.0", "", {}, "sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw=="], + + "yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], + + "yargs-parser": ["yargs-parser@21.1.1", "", {}, "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw=="], + + "@huggingface/transformers/onnxruntime-node": ["onnxruntime-node@1.21.0", "", { "dependencies": { "global-agent": "^3.0.0", "onnxruntime-common": "1.21.0", "tar": "^7.0.1" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-NeaCX6WW2L8cRCSqy3bInlo5ojjQqu2fD3D+9W5qb5irwxhEyWKXeH2vZ8W9r6VxaMPUan+4/7NDwZMtouZxEw=="], + + "chalk/supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], + + "filelist/minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="], + + "onnxruntime-web/onnxruntime-common": ["onnxruntime-common@1.22.0-dev.20250409-89f8206ba4", "", {}, "sha512-vDJMkfCfb0b1A836rgHj+ORuZf4B4+cc2bASQtpeoJLueuFc5DuYwjIZUBrSvx/fO5IrLjLz+oTrB3pcGlhovQ=="], + + "@huggingface/transformers/onnxruntime-node/onnxruntime-common": ["onnxruntime-common@1.21.0", "", {}, "sha512-Q632iLLrtCAVOTO65dh2+mNbQir/QNTVBG3h/QdZBpns7mZ0RYbLRBgGABPbpU9351AgYy7SJf1WaeVwMrBFPQ=="], + + "filelist/minimatch/brace-expansion": ["brace-expansion@2.0.1", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA=="], + } +} diff --git a/packages/crawler/db/bilibili_metadata.ts b/packages/crawler/db/bilibili_metadata.ts index f2a575a..acc136c 100644 --- a/packages/crawler/db/bilibili_metadata.ts +++ b/packages/crawler/db/bilibili_metadata.ts @@ -1,87 +1,77 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import type { Psql } from "global.d.ts"; import { AllDataType, BiliUserType } from "@core/db/schema"; -import Akari from "ml/akari.ts"; +import { AkariModelVersion } from "ml/const"; -export async function videoExistsInAllData(client: Client, aid: number) { - return await client.queryObject<{ exists: boolean }>( - `SELECT EXISTS(SELECT 1 FROM bilibili_metadata WHERE aid = $1)`, - [aid], - ) - .then((result) => result.rows[0].exists); +export async function videoExistsInAllData(sql: Psql, aid: number) { + const rows = await sql<{ exists: boolean }[]>` + SELECT EXISTS(SELECT 1 FROM bilibili_metadata WHERE aid = ${aid}) + `; + return rows[0].exists; } -export async function userExistsInBiliUsers(client: Client, uid: number) { - return await client.queryObject<{ exists: boolean }>(`SELECT EXISTS(SELECT 1 FROM bilibili_user WHERE uid = $1)`, [ - uid, - ]); +export async function userExistsInBiliUsers(sql: Psql, uid: number) { + const rows = await sql<{ exists: boolean }[]>` + SELECT EXISTS(SELECT 1 FROM bilibili_user WHERE uid = ${uid}) + `; + return rows[0].exists; } -export async function getUnlabelledVideos(client: Client) { - const queryResult = await client.queryObject<{ aid: number }>( - `SELECT a.aid FROM bilibili_metadata a LEFT JOIN labelling_result l ON a.aid = l.aid WHERE l.aid IS NULL`, - ); - return queryResult.rows.map((row) => row.aid); +export async function getUnlabelledVideos(sql: Psql) { + const rows = await sql<{ aid: number }[]>` + SELECT a.aid FROM bilibili_metadata a LEFT JOIN labelling_result l ON a.aid = l.aid WHERE l.aid IS NULL + `; + return rows.map((row) => row.aid); } -export async function insertVideoLabel(client: Client, aid: number, label: number) { - return await client.queryObject( - `INSERT INTO labelling_result (aid, label, model_version) VALUES ($1, $2, $3) ON CONFLICT (aid, model_version) DO NOTHING`, - [aid, label, Akari.getModelVersion()], - ); +export async function insertVideoLabel(sql: Psql, aid: number, label: number) { + await sql` + INSERT INTO labelling_result (aid, label, model_version) VALUES (${aid}, ${label}, ${AkariModelVersion}) ON CONFLICT (aid, model_version) DO NOTHING + `; } -export async function getVideoInfoFromAllData(client: Client, aid: number) { - const queryResult = await client.queryObject( - `SELECT * FROM bilibili_metadata WHERE aid = $1`, - [aid], - ); - const row = queryResult.rows[0]; - let authorInfo = ""; - if (row.uid && await userExistsInBiliUsers(client, row.uid)) { - const q = await client.queryObject( - `SELECT * FROM bilibili_user WHERE uid = $1`, - [row.uid], - ); - const userRow = q.rows[0]; - if (userRow) { - authorInfo = userRow.desc; - } - } - return { - title: row.title, - description: row.description, - tags: row.tags, - author_info: authorInfo, - }; +export async function getVideoInfoFromAllData(sql: Psql, aid: number) { + const rows = await sql` + SELECT * FROM bilibili_metadata WHERE aid = ${aid} + `; + const row = rows[0]; + let authorInfo = ""; + if (row.uid && await userExistsInBiliUsers(sql, row.uid)) { + const userRows = await sql` + SELECT * FROM bilibili_user WHERE uid = ${row.uid} + `; + const userRow = userRows[0]; + if (userRow) { + authorInfo = userRow.desc; + } + } + return { + title: row.title, + description: row.description, + tags: row.tags, + author_info: authorInfo, + }; } -export async function getUnArchivedBiliUsers(client: Client) { - const queryResult = await client.queryObject<{ uid: number }>( - ` - SELECT ad.uid - FROM bilibili_metadata ad - LEFT JOIN bilibili_user bu ON ad.uid = bu.uid - WHERE bu.uid IS NULL; - `, - [], - ); - const rows = queryResult.rows; - return rows.map((row) => row.uid); +export async function getUnArchivedBiliUsers(sql: Psql) { + const rows = await sql<{ uid: number }[]>` + SELECT ad.uid + FROM bilibili_metadata ad + LEFT JOIN bilibili_user bu ON ad.uid = bu.uid + WHERE bu.uid IS NULL; + `; + return rows.map((row) => row.uid); } -export async function setBiliVideoStatus(client: Client, aid: number, status: number) { - return await client.queryObject( - `UPDATE bilibili_metadata SET status = $1 WHERE aid = $2`, - [status, aid], - ); +export async function setBiliVideoStatus(sql: Psql, aid: number, status: number) { + await sql` + UPDATE bilibili_metadata SET status = ${status} WHERE aid = ${aid} + `; } -export async function getBiliVideoStatus(client: Client, aid: number) { - const queryResult = await client.queryObject<{ status: number }>( - `SELECT status FROM bilibili_metadata WHERE aid = $1`, - [aid], - ); - const rows = queryResult.rows; - if (rows.length === 0) return 0; - return rows[0].status; -} +export async function getBiliVideoStatus(sql: Psql, aid: number) { + const rows = await sql<{ status: number }[]>` + SELECT status FROM bilibili_metadata WHERE aid = ${aid} + `; + if (rows.length === 0) return 0; + return rows[0].status; +} \ No newline at end of file diff --git a/packages/crawler/db/init.ts b/packages/crawler/db/init.ts deleted file mode 100644 index 6bd1801..0000000 --- a/packages/crawler/db/init.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { postgresConfig } from "@core/db/pgConfig"; - -const pool = new Pool(postgresConfig, 12); - -export const db = pool; diff --git a/packages/crawler/db/snapshot.ts b/packages/crawler/db/snapshot.ts index 25c582d..b4635ee 100644 --- a/packages/crawler/db/snapshot.ts +++ b/packages/crawler/db/snapshot.ts @@ -1,9 +1,9 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { LatestSnapshotType } from "@core/db/schema"; import { SnapshotNumber } from "mq/task/getVideoStats.ts"; +import type { Psql } from "global.d.ts"; -export async function getVideosNearMilestone(client: Client) { - const queryResult = await client.queryObject(` +export async function getVideosNearMilestone(sql: Psql) { + const queryResult = await sql` SELECT ls.* FROM latest_video_snapshot ls RIGHT JOIN songs ON songs.aid = ls.aid @@ -18,8 +18,8 @@ export async function getVideosNearMilestone(client: Client) { (views >= 90000 AND views < 100000) OR (views >= 900000 AND views < 1000000) OR (views >= 9900000 AND views < 10000000) - `); - return queryResult.rows.map((row) => { + `; + return queryResult.map((row) => { return { ...row, aid: Number(row.aid), @@ -27,19 +27,16 @@ export async function getVideosNearMilestone(client: Client) { }); } -export async function getLatestVideoSnapshot(client: Client, aid: number): Promise { - const queryResult = await client.queryObject( - ` +export async function getLatestVideoSnapshot(sql: Psql, aid: number): Promise { + const queryResult = await sql` SELECT * FROM latest_video_snapshot - WHERE aid = $1 - `, - [aid], - ); - if (queryResult.rows.length === 0) { + WHERE aid = ${aid} + `; + if (queryResult.length === 0) { return null; } - return queryResult.rows.map((row) => { + return queryResult.map((row) => { return { ...row, aid: Number(row.aid), diff --git a/packages/crawler/db/snapshotSchedule.ts b/packages/crawler/db/snapshotSchedule.ts index 33afb26..9937a2d 100644 --- a/packages/crawler/db/snapshotSchedule.ts +++ b/packages/crawler/db/snapshotSchedule.ts @@ -1,10 +1,10 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { SnapshotScheduleType } from "@core/db/schema"; -import logger from "log/logger.ts"; -import { MINUTE } from "$std/datetime/constants.ts"; +import type { SnapshotScheduleType } from "@core/db/schema.d.ts"; +import logger from "@core/log/logger.ts"; +import { MINUTE } from "@core/const/time.ts"; import { redis } from "@core/db/redis.ts"; import { Redis } from "ioredis"; -import {parseTimestampFromPsql} from "../utils/formatTimestampToPostgre.ts"; +import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre.ts"; +import type { Psql } from "global.d.ts"; const REDIS_KEY = "cvsa:snapshot_window_counts"; @@ -14,11 +14,11 @@ function getCurrentWindowIndex(): number { return Math.floor(minutesSinceMidnight / 5); } -export async function refreshSnapshotWindowCounts(client: Client, redisClient: Redis) { +export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: Redis) { const now = new Date(); const startTime = now.getTime(); - const result = await client.queryObject<{ window_start: Date; count: number }>` + const result = await sql<{ window_start: Date; count: number }[]>` SELECT date_trunc('hour', started_at) + (EXTRACT(minute FROM started_at)::int / 5 * INTERVAL '5 minutes') AS window_start, @@ -33,7 +33,7 @@ export async function refreshSnapshotWindowCounts(client: Client, redisClient: R const currentWindow = getCurrentWindowIndex(); - for (const row of result.rows) { + for (const row of result) { const targetOffset = Math.floor((row.window_start.getTime() - startTime) / (5 * MINUTE)); const offset = currentWindow + targetOffset; if (offset >= 0) { @@ -42,10 +42,10 @@ export async function refreshSnapshotWindowCounts(client: Client, redisClient: R } } -export async function initSnapshotWindowCounts(client: Client, redisClient: Redis) { - await refreshSnapshotWindowCounts(client, redisClient); +export async function initSnapshotWindowCounts(sql: Psql, redisClient: Redis) { + await refreshSnapshotWindowCounts(sql, redisClient); setInterval(async () => { - await refreshSnapshotWindowCounts(client, redisClient); + await refreshSnapshotWindowCounts(sql, redisClient); }, 5 * MINUTE); } @@ -54,44 +54,56 @@ async function getWindowCount(redisClient: Redis, offset: number): Promise( - `SELECT id FROM snapshot_schedule WHERE id = $1`, - [id], - ); - return res.rows.length > 0; +export async function snapshotScheduleExists(sql: Psql, id: number) { + const rows = await sql<{ id: number }[]>` + SELECT id + FROM snapshot_schedule + WHERE id = ${id} + `; + return rows.length > 0; } -export async function videoHasActiveSchedule(client: Client, aid: number) { - const res = await client.queryObject<{ status: string }>( - `SELECT status FROM snapshot_schedule WHERE aid = $1 AND (status = 'pending' OR status = 'processing')`, - [aid], - ); - return res.rows.length > 0; +export async function videoHasActiveSchedule(sql: Psql, aid: number) { + const rows = await sql<{ status: string }[]>` + SELECT status + FROM snapshot_schedule + WHERE aid = ${aid} + AND (status = 'pending' + OR status = 'processing' + ) + ` + return rows.length > 0; } -export async function videoHasActiveScheduleWithType(client: Client, aid: number, type: string) { - const res = await client.queryObject<{ status: string }>( - `SELECT status FROM snapshot_schedule WHERE aid = $1 AND (status = 'pending' OR status = 'processing') AND type = $2`, - [aid, type], - ); - return res.rows.length > 0; +export async function videoHasActiveScheduleWithType(sql: Psql, aid: number, type: string) { + const rows = await sql<{ status: string }[]>` + SELECT status FROM snapshot_schedule + WHERE aid = ${aid} + AND (status = 'pending' OR status = 'processing') + AND type = ${type} + `; + return rows.length > 0; } -export async function videoHasProcessingSchedule(client: Client, aid: number) { - const res = await client.queryObject<{ status: string }>( - `SELECT status FROM snapshot_schedule WHERE aid = $1 AND status = 'processing'`, - [aid], - ); - return res.rows.length > 0; +export async function videoHasProcessingSchedule(sql: Psql, aid: number) { + const rows = await sql<{ status: string }[]>` + SELECT status + FROM snapshot_schedule + WHERE aid = ${aid} + AND status = 'processing' + ` + return rows.length > 0; } -export async function bulkGetVideosWithoutProcessingSchedules(client: Client, aids: number[]) { - const res = await client.queryObject<{ aid: number }>( - `SELECT aid FROM snapshot_schedule WHERE aid = ANY($1) AND status != 'processing' GROUP BY aid`, - [aids], - ); - return res.rows.map((row) => row.aid); +export async function bulkGetVideosWithoutProcessingSchedules(sql: Psql, aids: number[]) { + const rows = await sql<{ aid: string }[]>` + SELECT aid + FROM snapshot_schedule + WHERE aid = ANY(${aids}) + AND status != 'processing' + GROUP BY aid + ` + return rows.map((row) => Number(row.aid)); } interface Snapshot { @@ -99,118 +111,106 @@ interface Snapshot { views: number; } -export async function findClosestSnapshot( - client: Client, - aid: number, - targetTime: Date, -): Promise { - const query = ` +export async function findClosestSnapshot(sql: Psql, aid: number, targetTime: Date): Promise { + const result = await sql<{ created_at: string; views: number }[]>` SELECT created_at, views FROM video_snapshot - WHERE aid = $1 - ORDER BY ABS(EXTRACT(EPOCH FROM (created_at - $2::timestamptz))) + WHERE aid = ${aid} + ORDER BY ABS(EXTRACT(EPOCH FROM (created_at - ${targetTime.toISOString()}::timestamptz))) LIMIT 1 `; - const result = await client.queryObject<{ created_at: string; views: number }>( - query, - [aid, targetTime.toISOString()], - ); - if (result.rows.length === 0) return null; - const row = result.rows[0]; + if (result.length === 0) return null; + const row = result[0]; return { created_at: new Date(row.created_at).getTime(), - views: row.views, + views: row.views }; } -export async function findSnapshotBefore( - client: Client, - aid: number, - targetTime: Date, -): Promise { - const query = ` +export async function findSnapshotBefore(sql: Psql, aid: number, targetTime: Date): Promise { + const result = await sql<{ created_at: string; views: number }[]>` SELECT created_at, views FROM video_snapshot - WHERE aid = $1 - AND created_at <= $2::timestamptz + WHERE aid = ${aid} + AND created_at <= ${targetTime}::timestamptz ORDER BY created_at DESC LIMIT 1 `; - const result = await client.queryObject<{ created_at: string; views: number }>( - query, - [aid, targetTime.toISOString()], - ); - if (result.rows.length === 0) return null; - const row = result.rows[0]; + if (result.length === 0) return null; + const row = result[0]; return { created_at: new Date(row.created_at).getTime(), - views: row.views, + views: row.views }; } -export async function hasAtLeast2Snapshots(client: Client, aid: number) { - const res = await client.queryObject<{ count: number }>( - `SELECT COUNT(*) FROM video_snapshot WHERE aid = $1`, - [aid], - ); - return res.rows[0].count >= 2; +export async function hasAtLeast2Snapshots(sql: Psql, aid: number) { + const res = await sql<{ count: number }[]>` + SELECT COUNT(*) + FROM video_snapshot + WHERE aid = ${aid} + `; + return res[0].count >= 2; } -export async function getLatestSnapshot(client: Client, aid: number): Promise { - const res = await client.queryObject<{ created_at: string; views: number }>( - `SELECT created_at, views FROM video_snapshot WHERE aid = $1 ORDER BY created_at DESC LIMIT 1`, - [aid], - ); - if (res.rows.length === 0) return null; - const row = res.rows[0]; +export async function getLatestSnapshot(sql: Psql, aid: number): Promise { + const res = await sql<{ created_at: string; views: number }[]>` + SELECT created_at, views + FROM video_snapshot + WHERE aid = ${aid} + ORDER BY created_at DESC + LIMIT 1 + `; + if (res.length === 0) return null; + const row = res[0]; return { created_at: new Date(row.created_at).getTime(), - views: row.views, + views: row.views }; } -export async function getLatestActiveScheduleWithType(client: Client, aid: number, type: string) { - const query: string = ` +export async function getLatestActiveScheduleWithType(sql: Psql, aid: number, type: string) { + const rows = await sql` SELECT * FROM snapshot_schedule - WHERE aid = $1 - AND type = $2 + WHERE aid = ${aid} + AND type = ${type} AND (status = 'pending' OR status = 'processing') ORDER BY started_at DESC LIMIT 1 - ` - const res = await client.queryObject(query, [aid, type]); - return res.rows[0]; + `; + return rows[0]; } /* * Creates a new snapshot schedule record. - * @param client The database client. * @param aid The aid of the video. + * @param type Type of the snapshot. * @param targetTime Scheduled time for snapshot. (Timestamp in milliseconds) + * @param force Ignore all restrictions and force the creation of the schedule. */ export async function scheduleSnapshot( - client: Client, + sql: Psql, aid: number, type: string, targetTime: number, - force: boolean = false, + force: boolean = false ) { let adjustedTime = new Date(targetTime); - const hashActiveSchedule = await videoHasActiveScheduleWithType(client, aid, type); + const hashActiveSchedule = await videoHasActiveScheduleWithType(sql, aid, type); if (type == "milestone" && hashActiveSchedule) { - const latestActiveSchedule = await getLatestActiveScheduleWithType(client, aid, type); + const latestActiveSchedule = await getLatestActiveScheduleWithType(sql, aid, type); const latestScheduleStartedAt = new Date(parseTimestampFromPsql(latestActiveSchedule.started_at!)); if (latestScheduleStartedAt > adjustedTime) { - await client.queryObject(` + await sql` UPDATE snapshot_schedule - SET started_at = $1 - WHERE id = $2 - `, [adjustedTime, latestActiveSchedule.id]); + SET started_at = ${adjustedTime} + WHERE id = ${latestActiveSchedule.id} + `; logger.log( `Updated snapshot schedule for ${aid} at ${adjustedTime.toISOString()}`, "mq", - "fn:scheduleSnapshot", + "fn:scheduleSnapshot" ); return; } @@ -220,28 +220,33 @@ export async function scheduleSnapshot( adjustedTime = await adjustSnapshotTime(new Date(targetTime), 2000, redis); } logger.log(`Scheduled snapshot for ${aid} at ${adjustedTime.toISOString()}`, "mq", "fn:scheduleSnapshot"); - return client.queryObject( - `INSERT INTO snapshot_schedule (aid, type, started_at) VALUES ($1, $2, $3)`, - [aid, type, adjustedTime.toISOString()], - ); + return sql` + INSERT INTO snapshot_schedule + (aid, type, started_at) + VALUES ( + ${aid}, + ${type}, + ${adjustedTime.toISOString()} + ) + `; } export async function bulkScheduleSnapshot( - client: Client, + sql: Psql, aids: number[], type: string, targetTime: number, - force: boolean = false, + force: boolean = false ) { for (const aid of aids) { - await scheduleSnapshot(client, aid, type, targetTime, force); + await scheduleSnapshot(sql, aid, type, targetTime, force); } } export async function adjustSnapshotTime( expectedStartTime: Date, allowedCounts: number = 1000, - redisClient: Redis, + redisClient: Redis ): Promise { const currentWindow = getCurrentWindowIndex(); const targetOffset = Math.floor((expectedStartTime.getTime() - Date.now()) / (5 * MINUTE)) - 6; @@ -286,8 +291,8 @@ export async function adjustSnapshotTime( return expectedStartTime; } -export async function getSnapshotsInNextSecond(client: Client) { - const query = ` +export async function getSnapshotsInNextSecond(sql: Psql) { + const rows = await sql` SELECT * FROM snapshot_schedule WHERE started_at <= NOW() + INTERVAL '1 seconds' AND status = 'pending' AND type != 'normal' @@ -298,13 +303,12 @@ export async function getSnapshotsInNextSecond(client: Client) { END, started_at LIMIT 10; - `; - const res = await client.queryObject(query, []); - return res.rows; + ` + return rows; } -export async function getBulkSnapshotsInNextSecond(client: Client) { - const query = ` +export async function getBulkSnapshotsInNextSecond(sql: Psql) { + const rows = await sql` SELECT * FROM snapshot_schedule WHERE (started_at <= NOW() + INTERVAL '15 seconds') @@ -316,43 +320,38 @@ export async function getBulkSnapshotsInNextSecond(client: Client) { END, started_at LIMIT 1000; + ` + return rows; +} + +export async function setSnapshotStatus(sql: Psql, id: number, status: string) { + return await sql` + UPDATE snapshot_schedule SET status = ${status} WHERE id = ${id} `; - const res = await client.queryObject(query, []); - return res.rows; } -export async function setSnapshotStatus(client: Client, id: number, status: string) { - return await client.queryObject( - `UPDATE snapshot_schedule SET status = $2 WHERE id = $1`, - [id, status], - ); +export async function bulkSetSnapshotStatus(sql: Psql, ids: number[], status: string) { + return await sql` + UPDATE snapshot_schedule SET status = ${status} WHERE id = ANY(${ids}) + `; } -export async function bulkSetSnapshotStatus(client: Client, ids: number[], status: string) { - return await client.queryObject( - `UPDATE snapshot_schedule SET status = $2 WHERE id = ANY($1)`, - [ids, status], - ); -} - -export async function getVideosWithoutActiveSnapshotSchedule(client: Client) { - const query: string = ` +export async function getVideosWithoutActiveSnapshotSchedule(sql: Psql) { + const rows = await sql<{ aid: string }[]>` SELECT s.aid FROM songs s LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing') WHERE ss.aid IS NULL `; - const res = await client.queryObject<{ aid: number }>(query, []); - return res.rows.map((r) => Number(r.aid)); + return rows.map((r) => Number(r.aid)); } -export async function getAllVideosWithoutActiveSnapshotSchedule(client: Client) { - const query: string = ` +export async function getAllVideosWithoutActiveSnapshotSchedule(psql: Psql) { + const rows = await psql<{ aid: number }[]>` SELECT s.aid FROM bilibili_metadata s LEFT JOIN snapshot_schedule ss ON s.aid = ss.aid AND (ss.status = 'pending' OR ss.status = 'processing') WHERE ss.aid IS NULL - `; - const res = await client.queryObject<{ aid: number }>(query, []); - return res.rows.map((r) => Number(r.aid)); + ` + return rows.map((r) => Number(r.aid)); } diff --git a/packages/crawler/db/songs.ts b/packages/crawler/db/songs.ts index 1bfa002..5f5070f 100644 --- a/packages/crawler/db/songs.ts +++ b/packages/crawler/db/songs.ts @@ -1,45 +1,39 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import type { Psql } from "global.d.ts"; import { parseTimestampFromPsql } from "utils/formatTimestampToPostgre.ts"; -export async function getNotCollectedSongs(client: Client) { - const queryResult = await client.queryObject<{ aid: number }>(` - SELECT lr.aid - FROM labelling_result lr - WHERE lr.label != 0 - AND NOT EXISTS ( - SELECT 1 - FROM songs s - WHERE s.aid = lr.aid - ); - `); - return queryResult.rows.map((row) => row.aid); +export async function getNotCollectedSongs(sql: Psql) { + const rows = await sql<{ aid: number }[]>` + SELECT lr.aid + FROM labelling_result lr + WHERE lr.label != 0 + AND NOT EXISTS ( + SELECT 1 + FROM songs s + WHERE s.aid = lr.aid + ); + `; + return rows.map((row) => row.aid); } -export async function aidExistsInSongs(client: Client, aid: number) { - const queryResult = await client.queryObject<{ exists: boolean }>( - ` - SELECT EXISTS ( - SELECT 1 - FROM songs - WHERE aid = $1 - ); - `, - [aid], - ); - return queryResult.rows[0].exists; +export async function aidExistsInSongs(sql: Psql, aid: number) { + const rows = await sql<{ exists: boolean }[]>` + SELECT EXISTS ( + SELECT 1 + FROM songs + WHERE aid = ${aid} + ); + `; + return rows[0].exists; } -export async function getSongsPublihsedAt(client: Client, aid: number) { - const queryResult = await client.queryObject<{ published_at: string }>( - ` - SELECT published_at - FROM songs - WHERE aid = $1; - `, - [aid], - ); - if (queryResult.rows.length === 0) { - return null; - } - return parseTimestampFromPsql(queryResult.rows[0].published_at); -} +export async function getSongsPublihsedAt(sql: Psql, aid: number) { + const rows = await sql<{ published_at: string }[]>` + SELECT published_at + FROM songs + WHERE aid = ${aid}; + `; + if (rows.length === 0) { + return null; + } + return parseTimestampFromPsql(rows[0].published_at); +} \ No newline at end of file diff --git a/packages/crawler/db/withConnection.ts b/packages/crawler/db/withConnection.ts deleted file mode 100644 index a965f41..0000000 --- a/packages/crawler/db/withConnection.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { db } from "db/init.ts"; - -/** - * Executes a function with a database connection. - * @param operation The function that accepts the `client` as the parameter. - * @param errorHandling Optional function to handle errors. - * If no error handling function is provided, the error will be re-thrown. - * @param cleanup Optional function to execute after the operation. - * @returns The result of the operation or undefined if an error occurred. - */ -export async function withDbConnection( - operation: (client: Client) => Promise, - errorHandling?: (error: unknown, client: Client) => void, - cleanup?: () => void, -): Promise { - const client = await db.connect(); - try { - return await operation(client); - } catch (error) { - if (errorHandling) { - errorHandling(error, client); - return; - } - throw error; - } finally { - client.release(); - if (cleanup) { - cleanup(); - } - } -} diff --git a/packages/crawler/deno.json b/packages/crawler/deno.json deleted file mode 100644 index 1b1784d..0000000 --- a/packages/crawler/deno.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "@cvsa/crawler", - "tasks": { - "crawl-raw-bili": "deno --allow-env --allow-ffi --allow-read --allow-net --allow-write --allow-run src/db/raw/insertAidsToDB.ts", - "crawl-bili-aids": "deno --allow-env --allow-ffi --allow-read --allow-net --allow-write --allow-run src/db/raw/fetchAids.ts", - "check": "deno fmt --check && deno lint && deno check **/*.ts && deno check **/*.tsx", - "manifest": "deno task cli manifest $(pwd)", - "start": "deno run -A --watch=static/,routes/ dev.ts", - "build": "deno run -A dev.ts build", - "preview": "deno run -A main.ts", - "worker:main": "deno run --env-file=.env --allow-env --allow-read --allow-ffi --allow-net --allow-write --allow-run ./src/worker.ts", - "worker:filter": "deno run --env-file=.env --allow-env --allow-read --allow-ffi --allow-net --allow-write ./src/filterWorker.ts", - "adder": "deno run --env-file=.env --allow-env --allow-read --allow-ffi --allow-net ./src/jobAdder.ts", - "bullui": "deno run --allow-read --allow-env --allow-ffi --allow-net ./src/bullui.ts", - "all": "concurrently --restart-tries -1 'deno task worker:main' 'deno task adder' 'deno task bullui' 'deno task worker:filter'", - "test": "deno test ./test/ --allow-env --allow-ffi --allow-read --allow-net --allow-write --allow-run" - }, - "lint": { - "rules": { - "tags": ["recommended"] - } - }, - "imports": { - "@std/assert": "jsr:@std/assert@1", - "$std/": "https://deno.land/std@0.216.0/", - "@std/datetime": "jsr:@std/datetime@^0.225.4", - "@huggingface/transformers": "npm:@huggingface/transformers@3.0.0", - "bullmq": "npm:bullmq", - "mq/": "./mq/", - "db/": "./db/", - "@core/": "../core/", - "log/": "../core/log/", - "net/": "./net/", - "ml/": "./ml/", - "utils/": "./utils/", - "ioredis": "npm:ioredis", - "@bull-board/api": "npm:@bull-board/api", - "@bull-board/express": "npm:@bull-board/express", - "express": "npm:express", - "src/": "./src/", - "onnxruntime": "npm:onnxruntime-node@1.19.2", - "chalk": "npm:chalk", - "@core/db/schema": "../core/db/schema.d.ts", - "@core/db/pgConfig": "../core/db/pgConfig.ts" - }, - "exports": "./main.ts" -} diff --git a/packages/crawler/global.d.ts b/packages/crawler/global.d.ts new file mode 100644 index 0000000..37fc0e2 --- /dev/null +++ b/packages/crawler/global.d.ts @@ -0,0 +1,3 @@ +import type postgres from "postgres"; + +export type Psql = postgres.Sql<{}>; diff --git a/packages/crawler/ml/akari.ts b/packages/crawler/ml/akari.ts index 69a7a5d..d76317f 100644 --- a/packages/crawler/ml/akari.ts +++ b/packages/crawler/ml/akari.ts @@ -1,16 +1,17 @@ import { AIManager } from "ml/manager.ts"; -import * as ort from "onnxruntime"; -import logger from "log/logger.ts"; +import * as ort from "onnxruntime-node"; +import logger from "@core/log/logger.ts"; import { WorkerError } from "mq/schema.ts"; import { AutoTokenizer, PreTrainedTokenizer } from "@huggingface/transformers"; +import { AkariModelVersion } from "./const"; const tokenizerModel = "alikia2x/jina-embedding-v3-m2v-1024"; -const onnxClassifierPath = "../../model/akari/3.17.onnx"; +const onnxClassifierPath = `../../model/akari/${AkariModelVersion}.onnx`; const onnxEmbeddingPath = "../../model/embedding/model.onnx"; class AkariProto extends AIManager { private tokenizer: PreTrainedTokenizer | null = null; - private readonly modelVersion = "3.17"; + private readonly modelVersion = AkariModelVersion; constructor() { super(); @@ -104,4 +105,5 @@ class AkariProto extends AIManager { } const Akari = new AkariProto(); +await Akari.init(); export default Akari; diff --git a/packages/crawler/ml/const.ts b/packages/crawler/ml/const.ts new file mode 100644 index 0000000..4eb1525 --- /dev/null +++ b/packages/crawler/ml/const.ts @@ -0,0 +1 @@ +export const AkariModelVersion = "3.17"; \ No newline at end of file diff --git a/packages/crawler/ml/manager.ts b/packages/crawler/ml/manager.ts index 42f783e..d876844 100644 --- a/packages/crawler/ml/manager.ts +++ b/packages/crawler/ml/manager.ts @@ -1,5 +1,5 @@ -import * as ort from "onnxruntime"; -import logger from "log/logger.ts"; +import * as ort from "onnxruntime-node"; +import logger from "@core/log/logger.ts"; import { WorkerError } from "mq/schema.ts"; export class AIManager { diff --git a/packages/crawler/mq/exec/archiveSnapshots.ts b/packages/crawler/mq/exec/archiveSnapshots.ts index a35738a..91411a9 100644 --- a/packages/crawler/mq/exec/archiveSnapshots.ts +++ b/packages/crawler/mq/exec/archiveSnapshots.ts @@ -1,40 +1,40 @@ -import { Job } from "npm:bullmq@5.45.2"; +import { Job } from "bullmq"; import { getAllVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { lockManager } from "mq/lockManager.ts"; import { getLatestVideoSnapshot } from "db/snapshot.ts"; -import { HOUR, MINUTE } from "$std/datetime/constants.ts"; +import { HOUR, MINUTE } from "@core/const/time.ts"; +import { sql } from "@core/db/dbNew"; -export const archiveSnapshotsWorker = async (_job: Job) => - await withDbConnection(async (client: Client) => { +export const archiveSnapshotsWorker = async (_job: Job) => { + try { const startedAt = Date.now(); if (await lockManager.isLocked("dispatchArchiveSnapshots")) { logger.log("dispatchArchiveSnapshots is already running", "mq"); return; } await lockManager.acquireLock("dispatchArchiveSnapshots", 30 * 60); - const aids = await getAllVideosWithoutActiveSnapshotSchedule(client); + const aids = await getAllVideosWithoutActiveSnapshotSchedule(sql); for (const rawAid of aids) { const aid = Number(rawAid); - const latestSnapshot = await getLatestVideoSnapshot(client, aid); + const latestSnapshot = await getLatestVideoSnapshot(sql, aid); const now = Date.now(); const lastSnapshotedAt = latestSnapshot?.time ?? now; const interval = 168; logger.log( `Scheduled archive snapshot for aid ${aid} in ${interval} hours.`, "mq", - "fn:archiveSnapshotsWorker", + "fn:archiveSnapshotsWorker" ); const targetTime = lastSnapshotedAt + interval * HOUR; - await scheduleSnapshot(client, aid, "archive", targetTime); + await scheduleSnapshot(sql, aid, "archive", targetTime); if (now - startedAt > 250 * MINUTE) { return; } } - }, (e) => { + } catch (e) { logger.error(e as Error, "mq", "fn:archiveSnapshotsWorker"); - }, async () => { + } finally { await lockManager.releaseLock("dispatchArchiveSnapshots"); - }); + } +}; diff --git a/packages/crawler/mq/exec/classifyVideo.ts b/packages/crawler/mq/exec/classifyVideo.ts index 916a8ee..622eeed 100644 --- a/packages/crawler/mq/exec/classifyVideo.ts +++ b/packages/crawler/mq/exec/classifyVideo.ts @@ -1,23 +1,22 @@ import { Job } from "bullmq"; -import { db } from "db/init.ts"; import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel } from "../../db/bilibili_metadata.ts"; import Akari from "ml/akari.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { lockManager } from "mq/lockManager.ts"; import { aidExistsInSongs } from "db/songs.ts"; import { insertIntoSongs } from "mq/task/collectSongs.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import { MINUTE } from "@std/datetime"; +import { MINUTE } from "@core/const/time.ts"; +import { sql } from "@core/db/dbNew.ts"; export const classifyVideoWorker = async (job: Job) => { - const client = await db.connect(); const aid = job.data.aid; if (!aid) { return 3; } - const videoInfo = await getVideoInfoFromAllData(client, aid); + const videoInfo = await getVideoInfoFromAllData(sql, aid); const title = videoInfo.title?.trim() || "untitled"; const description = videoInfo.description?.trim() || "N/A"; const tags = videoInfo.tags?.trim() || "empty"; @@ -25,16 +24,14 @@ export const classifyVideoWorker = async (job: Job) => { if (label == -1) { logger.warn(`Failed to classify video ${aid}`, "ml"); } - await insertVideoLabel(client, aid, label); + await insertVideoLabel(sql, aid, label); - const exists = await aidExistsInSongs(client, aid); + const exists = await aidExistsInSongs(sql, aid); if (!exists && label !== 0) { - await scheduleSnapshot(client, aid, "new", Date.now() + 10 * MINUTE, true); - await insertIntoSongs(client, aid); + await scheduleSnapshot(sql, aid, "new", Date.now() + 10 * MINUTE, true); + await insertIntoSongs(sql, aid); } - client.release(); - await job.updateData({ ...job.data, label: label, @@ -51,10 +48,8 @@ export const classifyVideosWorker = async () => { await lockManager.acquireLock("classifyVideos"); - const client = await db.connect(); - const videos = await getUnlabelledVideos(client); + const videos = await getUnlabelledVideos(sql); logger.log(`Found ${videos.length} unlabelled videos`); - client.release(); let i = 0; for (const aid of videos) { diff --git a/packages/crawler/mq/exec/collectSongs.ts b/packages/crawler/mq/exec/collectSongs.ts index 1e4db3c..bd58179 100644 --- a/packages/crawler/mq/exec/collectSongs.ts +++ b/packages/crawler/mq/exec/collectSongs.ts @@ -1,9 +1,7 @@ -import { Job } from "npm:bullmq@5.45.2"; +import { Job } from "bullmq"; import { collectSongs } from "mq/task/collectSongs.ts"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -export const collectSongsWorker = (_job: Job): Promise => - withDbConnection(async (client: Client) => { - await collectSongs(client); - }); +export const collectSongsWorker = async (_job: Job): Promise =>{ + await collectSongs(); + return; +} \ No newline at end of file diff --git a/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts b/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts index 1d8d8c7..2cbcd08 100644 --- a/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchMilestoneSnapshots.ts @@ -1,19 +1,18 @@ -import { Job } from "npm:bullmq@5.45.2"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import { Job } from "bullmq"; import { getVideosNearMilestone } from "db/snapshot.ts"; import { getAdjustedShortTermETA } from "mq/scheduling.ts"; import { truncate } from "utils/truncate.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import logger from "log/logger.ts"; -import { HOUR, MINUTE, SECOND } from "@std/datetime"; +import logger from "@core/log/logger.ts"; +import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; +import { sql } from "@core/db/dbNew"; -export const dispatchMilestoneSnapshotsWorker = (_job: Job): Promise => - withDbConnection(async (client: Client) => { - const videos = await getVideosNearMilestone(client); +export const dispatchMilestoneSnapshotsWorker = async (_job: Job) => { + try { + const videos = await getVideosNearMilestone(sql); for (const video of videos) { const aid = Number(video.aid); - const eta = await getAdjustedShortTermETA(client, aid); + const eta = await getAdjustedShortTermETA(sql, aid); if (eta > 144) continue; const now = Date.now(); const scheduledNextSnapshotDelay = eta * HOUR; @@ -21,9 +20,10 @@ export const dispatchMilestoneSnapshotsWorker = (_job: Job): Promise => const minInterval = 1 * SECOND; const delay = truncate(scheduledNextSnapshotDelay, minInterval, maxInterval); const targetTime = now + delay; - await scheduleSnapshot(client, aid, "milestone", targetTime); + await scheduleSnapshot(sql, aid, "milestone", targetTime); logger.log(`Scheduled milestone snapshot for aid ${aid} in ${(delay / MINUTE).toFixed(2)} mins.`, "mq"); } - }, (e) => { + } catch (e) { logger.error(e as Error, "mq", "fn:dispatchMilestoneSnapshotsWorker"); - }); + }; +} \ No newline at end of file diff --git a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts index be9e02c..5c1652f 100644 --- a/packages/crawler/mq/exec/dispatchRegularSnapshots.ts +++ b/packages/crawler/mq/exec/dispatchRegularSnapshots.ts @@ -1,16 +1,15 @@ -import { Job } from "npm:bullmq@5.45.2"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import { Job } from "bullmq"; import { getLatestVideoSnapshot } from "db/snapshot.ts"; import { truncate } from "utils/truncate.ts"; import { getVideosWithoutActiveSnapshotSchedule, scheduleSnapshot } from "db/snapshotSchedule.ts"; -import logger from "log/logger.ts"; -import { HOUR, MINUTE, WEEK } from "@std/datetime"; -import { lockManager } from "../lockManager.ts"; -import { getRegularSnapshotInterval } from "../task/regularSnapshotInterval.ts"; +import logger from "@core/log/logger.ts"; +import { HOUR, MINUTE, WEEK } from "@core/const/time.ts"; +import { lockManager } from "mq/lockManager.ts"; +import { getRegularSnapshotInterval } from "mq/task/regularSnapshotInterval.ts"; +import { sql } from "@core/db/dbNew.ts"; -export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise => - await withDbConnection(async (client: Client) => { +export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise => { + try { const startedAt = Date.now(); if (await lockManager.isLocked("dispatchRegularSnapshots")) { logger.log("dispatchRegularSnapshots is already running", "mq"); @@ -18,22 +17,23 @@ export const dispatchRegularSnapshotsWorker = async (_job: Job): Promise = } await lockManager.acquireLock("dispatchRegularSnapshots", 30 * 60); - const aids = await getVideosWithoutActiveSnapshotSchedule(client); + const aids = await getVideosWithoutActiveSnapshotSchedule(sql); for (const rawAid of aids) { const aid = Number(rawAid); - const latestSnapshot = await getLatestVideoSnapshot(client, aid); + const latestSnapshot = await getLatestVideoSnapshot(sql, aid); const now = Date.now(); const lastSnapshotedAt = latestSnapshot?.time ?? now; - const interval = await getRegularSnapshotInterval(client, aid); + const interval = await getRegularSnapshotInterval(sql, aid); logger.log(`Scheduled regular snapshot for aid ${aid} in ${interval} hours.`, "mq"); const targetTime = truncate(lastSnapshotedAt + interval * HOUR, now + 1, now + 100000 * WEEK); - await scheduleSnapshot(client, aid, "normal", targetTime); + await scheduleSnapshot(sql, aid, "normal", targetTime); if (now - startedAt > 25 * MINUTE) { return; } } - }, (e) => { + } catch (e) { logger.error(e as Error, "mq", "fn:regularSnapshotsWorker"); - }, async () => { + } finally { await lockManager.releaseLock("dispatchRegularSnapshots"); - }); + } +}; diff --git a/packages/crawler/mq/exec/getLatestVideos.ts b/packages/crawler/mq/exec/getLatestVideos.ts index b05d393..b3d93f1 100644 --- a/packages/crawler/mq/exec/getLatestVideos.ts +++ b/packages/crawler/mq/exec/getLatestVideos.ts @@ -1,9 +1,7 @@ +import { sql } from "@core/db/dbNew"; import { Job } from "bullmq"; import { queueLatestVideos } from "mq/task/queueLatestVideo.ts"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -export const getLatestVideosWorker = (_job: Job): Promise => - withDbConnection(async (client: Client) => { - await queueLatestVideos(client); - }); +export const getLatestVideosWorker = async (_job: Job): Promise =>{ + await queueLatestVideos(sql); +} diff --git a/packages/crawler/mq/exec/getVideoInfo.ts b/packages/crawler/mq/exec/getVideoInfo.ts index 2ed2a8a..ee96b2d 100644 --- a/packages/crawler/mq/exec/getVideoInfo.ts +++ b/packages/crawler/mq/exec/getVideoInfo.ts @@ -1,15 +1,13 @@ -import { Job } from "npm:bullmq@5.45.2"; +import { Job } from "bullmq"; import { insertVideoInfo } from "mq/task/getVideoDetails.ts"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; +import { sql } from "@core/db/dbNew"; -export const getVideoInfoWorker = async (job: Job): Promise => - await withDbConnection(async (client: Client) => { - const aid = job.data.aid; - if (!aid) { - logger.warn("aid does not exists", "mq", "job:getVideoInfo"); - return; - } - await insertVideoInfo(client, aid); - }); +export const getVideoInfoWorker = async (job: Job): Promise => { + const aid = job.data.aid; + if (!aid) { + logger.warn("aid does not exists", "mq", "job:getVideoInfo"); + return; + } + await insertVideoInfo(sql, aid); +} diff --git a/packages/crawler/mq/exec/scheduleCleanup.ts b/packages/crawler/mq/exec/scheduleCleanup.ts index 920458e..821abe4 100644 --- a/packages/crawler/mq/exec/scheduleCleanup.ts +++ b/packages/crawler/mq/exec/scheduleCleanup.ts @@ -1,20 +1,25 @@ -import { Job } from "npm:bullmq@5.45.2"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import logger from "log/logger.ts"; +import { Job } from "bullmq"; +import { sql } from "@core/db/dbNew"; +import logger from "@core/log/logger.ts"; import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule.ts"; -import { SECOND } from "@std/datetime"; +import { SECOND } from "@core/const/time.ts"; import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount.ts"; import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules.ts"; -export const scheduleCleanupWorker = async (_job: Job): Promise => - await withDbConnection(async (client: Client) => { - if (await getTimeoutSchedulesCount(client) > 2000) { - await removeAllTimeoutSchedules(client); +interface SnapshotSchedule { + id: bigint; + aid: bigint; + type: string; +} + +export const scheduleCleanupWorker = async (_job: Job): Promise => { + try { + if ((await getTimeoutSchedulesCount()) > 2000) { + await removeAllTimeoutSchedules(); return; } - const query: string = ` + const rows = await sql` SELECT id, aid, type FROM snapshot_schedule WHERE status IN ('pending', 'processing') @@ -26,20 +31,21 @@ export const scheduleCleanupWorker = async (_job: Job): Promise => AND started_at < NOW() - INTERVAL '2 minutes' AND type = 'milestone' `; - const { rows } = await client.queryObject<{ id: bigint; aid: bigint; type: string }>(query); + if (rows.length === 0) return; for (const row of rows) { const id = Number(row.id); const aid = Number(row.aid); const type = row.type; - await setSnapshotStatus(client, id, "timeout"); - await scheduleSnapshot(client, aid, type, Date.now() + 10 * SECOND); + await setSnapshotStatus(sql, id, "timeout"); + await scheduleSnapshot(sql, aid, type, Date.now() + 10 * SECOND); logger.log( `Schedule ${id} has not received any response in a while, rescheduled.`, "mq", - "fn:scheduleCleanupWorker", + "fn:scheduleCleanupWorker" ); } - }, (e) => { + } catch (e) { logger.error(e as Error, "mq", "fn:scheduleCleanupWorker"); - }); + } +}; diff --git a/packages/crawler/mq/exec/snapshotTick.ts b/packages/crawler/mq/exec/snapshotTick.ts index 74b7598..d599fca 100644 --- a/packages/crawler/mq/exec/snapshotTick.ts +++ b/packages/crawler/mq/exec/snapshotTick.ts @@ -1,5 +1,4 @@ import { Job } from "bullmq"; -import { db } from "db/init.ts"; import { bulkGetVideosWithoutProcessingSchedules, bulkSetSnapshotStatus, @@ -8,8 +7,9 @@ import { setSnapshotStatus, videoHasProcessingSchedule, } from "db/snapshotSchedule.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { SnapshotQueue } from "mq/index.ts"; +import { sql } from "@core/db/dbNew"; const priorityMap: { [key: string]: number } = { "milestone": 1, @@ -17,17 +17,16 @@ const priorityMap: { [key: string]: number } = { }; export const bulkSnapshotTickWorker = async (_job: Job) => { - const client = await db.connect(); try { - const schedules = await getBulkSnapshotsInNextSecond(client); + const schedules = await getBulkSnapshotsInNextSecond(sql); const count = schedules.length; const groups = Math.ceil(count / 30); for (let i = 0; i < groups; i++) { const group = schedules.slice(i * 30, (i + 1) * 30); const aids = group.map((schedule) => Number(schedule.aid)); - const filteredAids = await bulkGetVideosWithoutProcessingSchedules(client, aids); + const filteredAids = await bulkGetVideosWithoutProcessingSchedules(sql, aids); if (filteredAids.length === 0) continue; - await bulkSetSnapshotStatus(client, filteredAids, "processing"); + await bulkSetSnapshotStatus(sql, filteredAids, "processing"); const schedulesData = group.map((schedule) => { return { aid: Number(schedule.aid), @@ -46,17 +45,14 @@ export const bulkSnapshotTickWorker = async (_job: Job) => { return `OK`; } catch (e) { logger.error(e as Error); - } finally { - client.release(); } }; export const snapshotTickWorker = async (_job: Job) => { - const client = await db.connect(); try { - const schedules = await getSnapshotsInNextSecond(client); + const schedules = await getSnapshotsInNextSecond(sql); for (const schedule of schedules) { - if (await videoHasProcessingSchedule(client, Number(schedule.aid))) { + if (await videoHasProcessingSchedule(sql, Number(schedule.aid))) { continue; } let priority = 3; @@ -64,7 +60,7 @@ export const snapshotTickWorker = async (_job: Job) => { priority = priorityMap[schedule.type]; } const aid = Number(schedule.aid); - await setSnapshotStatus(client, schedule.id, "processing"); + await setSnapshotStatus(sql, schedule.id, "processing"); await SnapshotQueue.add("snapshotVideo", { aid: Number(aid), id: Number(schedule.id), @@ -74,8 +70,6 @@ export const snapshotTickWorker = async (_job: Job) => { return `OK`; } catch (e) { logger.error(e as Error); - } finally { - client.release(); } }; diff --git a/packages/crawler/mq/exec/snapshotVideo.ts b/packages/crawler/mq/exec/snapshotVideo.ts index ffc3fd3..261b78b 100644 --- a/packages/crawler/mq/exec/snapshotVideo.ts +++ b/packages/crawler/mq/exec/snapshotVideo.ts @@ -1,15 +1,14 @@ -import { Job } from "npm:bullmq@5.45.2"; -import { withDbConnection } from "db/withConnection.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import { Job } from "bullmq"; import { scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule.ts"; -import logger from "log/logger.ts"; -import { HOUR, MINUTE, SECOND } from "@std/datetime"; +import logger from "@core/log/logger.ts"; +import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; import { lockManager } from "mq/lockManager.ts"; import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata.ts"; import { insertVideoSnapshot } from "mq/task/getVideoStats.ts"; import { getSongsPublihsedAt } from "db/songs.ts"; import { getAdjustedShortTermETA } from "mq/scheduling.ts"; import { NetSchedulerError } from "@core/net/delegate.ts"; +import { sql } from "@core/db/dbNew.ts"; const snapshotTypeToTaskMap: { [key: string]: string } = { "milestone": "snapshotMilestoneVideo", @@ -23,12 +22,12 @@ export const snapshotVideoWorker = async (job: Job): Promise => { const type = job.data.type; const task = snapshotTypeToTaskMap[type] ?? "snapshotVideo"; const retryInterval = type === "milestone" ? 5 * SECOND : 2 * MINUTE; - await withDbConnection(async (client: Client) => { - const exists = await snapshotScheduleExists(client, id); + try { + const exists = await snapshotScheduleExists(sql, id); if (!exists) { return; } - const status = await getBiliVideoStatus(client, aid); + const status = await getBiliVideoStatus(sql, aid); if (status !== 0) { logger.warn( `Video ${aid} has status ${status} in the database. Abort snapshoting.`, @@ -37,11 +36,11 @@ export const snapshotVideoWorker = async (job: Job): Promise => { ); return; } - await setSnapshotStatus(client, id, "processing"); - const stat = await insertVideoSnapshot(client, aid, task); + await setSnapshotStatus(sql, id, "processing"); + const stat = await insertVideoSnapshot(sql, aid, task); if (typeof stat === "number") { - await setBiliVideoStatus(client, aid, stat); - await setSnapshotStatus(client, id, "bili_error"); + await setBiliVideoStatus(sql, aid, stat); + await setSnapshotStatus(sql, id, "bili_error"); logger.warn( `Bilibili return status ${status} when snapshoting for ${aid}.`, "mq", @@ -49,9 +48,9 @@ export const snapshotVideoWorker = async (job: Job): Promise => { ); return; } - await setSnapshotStatus(client, id, "completed"); + await setSnapshotStatus(sql, id, "completed"); if (type === "new") { - const publihsedAt = await getSongsPublihsedAt(client, aid); + const publihsedAt = await getSongsPublihsedAt(sql, aid); const timeSincePublished = stat.time - publihsedAt!; const viewsPerHour = stat.views / timeSincePublished * HOUR; if (timeSincePublished > 48 * HOUR) { @@ -70,10 +69,10 @@ export const snapshotVideoWorker = async (job: Job): Promise => { if (viewsPerHour > 1000) { intervalMins = 15; } - await scheduleSnapshot(client, aid, type, Date.now() + intervalMins * MINUTE, true); + await scheduleSnapshot(sql, aid, type, Date.now() + intervalMins * MINUTE, true); } if (type !== "milestone") return; - const eta = await getAdjustedShortTermETA(client, aid); + const eta = await getAdjustedShortTermETA(sql, aid); if (eta > 144) { const etaHoursString = eta.toFixed(2) + " hrs"; logger.warn( @@ -84,18 +83,19 @@ export const snapshotVideoWorker = async (job: Job): Promise => { } const now = Date.now(); const targetTime = now + eta * HOUR; - await scheduleSnapshot(client, aid, type, targetTime); - await setSnapshotStatus(client, id, "completed"); + await scheduleSnapshot(sql, aid, type, targetTime); + await setSnapshotStatus(sql, id, "completed"); return; - }, async (e, client) => { + } + catch (e) { if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") { logger.warn( `No available proxy for aid ${job.data.aid}.`, "mq", "fn:takeSnapshotForVideoWorker", ); - await setSnapshotStatus(client, id, "no_proxy"); - await scheduleSnapshot(client, aid, type, Date.now() + retryInterval); + await setSnapshotStatus(sql, id, "no_proxy"); + await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval); return; } else if (e instanceof NetSchedulerError && e.code === "ALICLOUD_PROXY_ERR") { @@ -104,13 +104,14 @@ export const snapshotVideoWorker = async (job: Job): Promise => { "mq", "fn:takeSnapshotForVideoWorker", ); - await setSnapshotStatus(client, id, "failed"); - await scheduleSnapshot(client, aid, type, Date.now() + retryInterval); + await setSnapshotStatus(sql, id, "failed"); + await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval); } logger.error(e as Error, "mq", "fn:takeSnapshotForVideoWorker"); - await setSnapshotStatus(client, id, "failed"); - }, async () => { + await setSnapshotStatus(sql, id, "failed"); + } + finally { await lockManager.releaseLock("dispatchRegularSnapshots"); - }); + }; return; }; diff --git a/packages/crawler/mq/exec/takeBulkSnapshot.ts b/packages/crawler/mq/exec/takeBulkSnapshot.ts index eb27119..a71add1 100644 --- a/packages/crawler/mq/exec/takeBulkSnapshot.ts +++ b/packages/crawler/mq/exec/takeBulkSnapshot.ts @@ -1,5 +1,4 @@ -import { Job } from "npm:bullmq@5.45.2"; -import { db } from "db/init.ts"; +import { Job } from "bullmq"; import { bulkScheduleSnapshot, bulkSetSnapshotStatus, @@ -7,22 +6,22 @@ import { snapshotScheduleExists, } from "db/snapshotSchedule.ts"; import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { NetSchedulerError } from "@core/net/delegate.ts"; -import { HOUR, MINUTE, SECOND } from "@std/datetime"; +import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; import { getRegularSnapshotInterval } from "../task/regularSnapshotInterval.ts"; import { SnapshotScheduleType } from "@core/db/schema"; +import { sql } from "@core/db/dbNew.ts"; export const takeBulkSnapshotForVideosWorker = async (job: Job) => { const schedules: SnapshotScheduleType[] = job.data.schedules; const ids = schedules.map((schedule) => Number(schedule.id)); const aidsToFetch: number[] = []; - const client = await db.connect(); try { for (const schedule of schedules) { const aid = Number(schedule.aid); const id = Number(schedule.id); - const exists = await snapshotScheduleExists(client, id); + const exists = await snapshotScheduleExists(sql, id); if (!exists) { continue; } @@ -30,8 +29,8 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => { } const data = await bulkGetVideoStats(aidsToFetch); if (typeof data === "number") { - await bulkSetSnapshotStatus(client, ids, "failed"); - await bulkScheduleSnapshot(client, aidsToFetch, "normal", Date.now() + 15 * SECOND); + await bulkSetSnapshotStatus(sql, ids, "failed"); + await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 15 * SECOND); return `GET_BILI_STATUS_${data}`; } for (const video of data) { @@ -44,26 +43,31 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => { const coins = stat.coin; const shares = stat.share; const favorites = stat.collect; - const query: string = ` + await sql` INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - `; - await client.queryObject( - query, - [aid, views, danmakus, replies, likes, coins, shares, favorites], - ); + VALUES ( + ${aid}, + ${views}, + ${danmakus}, + ${replies}, + ${likes}, + ${coins}, + ${shares}, + ${favorites} + ) + ` logger.log(`Taken snapshot for video ${aid} in bulk.`, "net", "fn:takeBulkSnapshotForVideosWorker"); } - await bulkSetSnapshotStatus(client, ids, "completed"); + await bulkSetSnapshotStatus(sql, ids, "completed"); for (const schedule of schedules) { const aid = Number(schedule.aid); const type = schedule.type; if (type == "archive") continue; - const interval = await getRegularSnapshotInterval(client, aid); + const interval = await getRegularSnapshotInterval(sql, aid); logger.log(`Scheduled regular snapshot for aid ${aid} in ${interval} hours.`, "mq"); - await scheduleSnapshot(client, aid, "normal", Date.now() + interval * HOUR); + await scheduleSnapshot(sql, aid, "normal", Date.now() + interval * HOUR); } return `DONE`; } catch (e) { @@ -73,13 +77,11 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => { "mq", "fn:takeBulkSnapshotForVideosWorker", ); - await bulkSetSnapshotStatus(client, ids, "no_proxy"); - await bulkScheduleSnapshot(client, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random()); + await bulkSetSnapshotStatus(sql, ids, "no_proxy"); + await bulkScheduleSnapshot(sql, aidsToFetch, "normal", Date.now() + 20 * MINUTE * Math.random()); return; } logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker"); - await bulkSetSnapshotStatus(client, ids, "failed"); - } finally { - client.release(); + await bulkSetSnapshotStatus(sql, ids, "failed"); } }; diff --git a/packages/crawler/mq/init.ts b/packages/crawler/mq/init.ts index 518afe4..f646c08 100644 --- a/packages/crawler/mq/init.ts +++ b/packages/crawler/mq/init.ts @@ -1,72 +1,75 @@ -import { HOUR, MINUTE, SECOND } from "$std/datetime/constants.ts"; +import { HOUR, MINUTE, SECOND } from "@core/const/time.ts"; import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { initSnapshotWindowCounts } from "db/snapshotSchedule.ts"; -import { db } from "db/init.ts"; import { redis } from "@core/db/redis.ts"; +import { sql } from "@core/db/dbNew"; export async function initMQ() { - const client = await db.connect(); - try { - await initSnapshotWindowCounts(client, redis); + await initSnapshotWindowCounts(sql, redis); - await LatestVideosQueue.upsertJobScheduler("getLatestVideos", { - every: 1 * MINUTE, - immediately: true, - }); + await LatestVideosQueue.upsertJobScheduler("getLatestVideos", { + every: 1 * MINUTE, + immediately: true + }); - await ClassifyVideoQueue.upsertJobScheduler("classifyVideos", { - every: 5 * MINUTE, - immediately: true, - }); + await ClassifyVideoQueue.upsertJobScheduler("classifyVideos", { + every: 5 * MINUTE, + immediately: true + }); - await LatestVideosQueue.upsertJobScheduler("collectSongs", { - every: 3 * MINUTE, - immediately: true, - }); + await LatestVideosQueue.upsertJobScheduler("collectSongs", { + every: 3 * MINUTE, + immediately: true + }); - await SnapshotQueue.upsertJobScheduler("snapshotTick", { + await SnapshotQueue.upsertJobScheduler( + "snapshotTick", + { every: 1 * SECOND, - immediately: true, - }, { + immediately: true + }, + { opts: { removeOnComplete: 300, - removeOnFail: 600, - }, - }); + removeOnFail: 600 + } + } + ); - await SnapshotQueue.upsertJobScheduler("bulkSnapshotTick", { + await SnapshotQueue.upsertJobScheduler( + "bulkSnapshotTick", + { every: 15 * SECOND, - immediately: true, - }, { + immediately: true + }, + { opts: { removeOnComplete: 60, - removeOnFail: 600, - }, - }); + removeOnFail: 600 + } + } + ); - await SnapshotQueue.upsertJobScheduler("dispatchMilestoneSnapshots", { - every: 5 * MINUTE, - immediately: true, - }); + await SnapshotQueue.upsertJobScheduler("dispatchMilestoneSnapshots", { + every: 5 * MINUTE, + immediately: true + }); - await SnapshotQueue.upsertJobScheduler("dispatchRegularSnapshots", { - every: 30 * MINUTE, - immediately: true, - }); + await SnapshotQueue.upsertJobScheduler("dispatchRegularSnapshots", { + every: 30 * MINUTE, + immediately: true + }); - await SnapshotQueue.upsertJobScheduler("dispatchArchiveSnapshots", { - every: 6 * HOUR, - immediately: true, - }); + await SnapshotQueue.upsertJobScheduler("dispatchArchiveSnapshots", { + every: 6 * HOUR, + immediately: true + }); - await SnapshotQueue.upsertJobScheduler("scheduleCleanup", { - every: 2 * MINUTE, - immediately: true, - }); + await SnapshotQueue.upsertJobScheduler("scheduleCleanup", { + every: 2 * MINUTE, + immediately: true + }); - logger.log("Message queue initialized."); - } finally { - client.release(); - } + logger.log("Message queue initialized."); } diff --git a/packages/crawler/mq/scheduling.ts b/packages/crawler/mq/scheduling.ts index e72596e..b84d73f 100644 --- a/packages/crawler/mq/scheduling.ts +++ b/packages/crawler/mq/scheduling.ts @@ -1,8 +1,8 @@ import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts"; import { truncate } from "utils/truncate.ts"; import { closetMilestone } from "./exec/snapshotTick.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { HOUR, MINUTE } from "@std/datetime"; +import { HOUR, MINUTE } from "@core/const/time.ts"; +import type { Psql } from "global.d.ts"; const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base); @@ -26,11 +26,11 @@ const getFactor = (x: number) => { * @param aid - aid of the video * @returns ETA in hours */ -export const getAdjustedShortTermETA = async (client: Client, aid: number) => { - const latestSnapshot = await getLatestSnapshot(client, aid); +export const getAdjustedShortTermETA = async (sql: Psql, aid: number) => { + const latestSnapshot = await getLatestSnapshot(sql, aid); // Immediately dispatch a snapshot if there is no snapshot yet if (!latestSnapshot) return 0; - const snapshotsEnough = await hasAtLeast2Snapshots(client, aid); + const snapshotsEnough = await hasAtLeast2Snapshots(sql, aid); if (!snapshotsEnough) return 0; const currentTimestamp = new Date().getTime(); @@ -40,7 +40,7 @@ export const getAdjustedShortTermETA = async (client: Client, aid: number) => { for (const timeInterval of timeIntervals) { const date = new Date(currentTimestamp - timeInterval); - const snapshot = await findClosestSnapshot(client, aid, date); + const snapshot = await findClosestSnapshot(sql, aid, date); if (!snapshot) continue; const hoursDiff = (latestSnapshot.created_at - snapshot.created_at) / HOUR; const viewsDiff = latestSnapshot.views - snapshot.views; diff --git a/packages/crawler/mq/task/collectSongs.ts b/packages/crawler/mq/task/collectSongs.ts index ca186f2..1d7634d 100644 --- a/packages/crawler/mq/task/collectSongs.ts +++ b/packages/crawler/mq/task/collectSongs.ts @@ -1,31 +1,29 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import { sql } from "@core/db/dbNew"; import { aidExistsInSongs, getNotCollectedSongs } from "db/songs.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { scheduleSnapshot } from "db/snapshotSchedule.ts"; -import { MINUTE } from "@std/datetime"; +import { MINUTE } from "@core/const/time.ts"; +import type { Psql } from "global.d.ts"; -export async function collectSongs(client: Client) { - const aids = await getNotCollectedSongs(client); +export async function collectSongs() { + const aids = await getNotCollectedSongs(sql); for (const aid of aids) { - const exists = await aidExistsInSongs(client, aid); + const exists = await aidExistsInSongs(sql, aid); if (exists) continue; - await insertIntoSongs(client, aid); - await scheduleSnapshot(client, aid, "new", Date.now() + 10 * MINUTE, true); + await insertIntoSongs(sql, aid); + await scheduleSnapshot(sql, aid, "new", Date.now() + 10 * MINUTE, true); logger.log(`Video ${aid} was added into the songs table.`, "mq", "fn:collectSongs"); } } -export async function insertIntoSongs(client: Client, aid: number) { - await client.queryObject( - ` - INSERT INTO songs (aid, published_at, duration) - VALUES ( - $1, - (SELECT published_at FROM bilibili_metadata WHERE aid = $1), - (SELECT duration FROM bilibili_metadata WHERE aid = $1) - ) - ON CONFLICT DO NOTHING - `, - [aid], - ); +export async function insertIntoSongs(sql: Psql, aid: number) { + await sql` + INSERT INTO songs (aid, published_at, duration) + VALUES ( + $1, + (SELECT published_at FROM bilibili_metadata WHERE aid = ${aid}), + (SELECT duration FROM bilibili_metadata WHERE aid = ${aid}) + ) + ON CONFLICT DO NOTHING + ` } diff --git a/packages/crawler/mq/task/getTimeoutSchedulesCount.ts b/packages/crawler/mq/task/getTimeoutSchedulesCount.ts index 4e7bb14..e66691f 100644 --- a/packages/crawler/mq/task/getTimeoutSchedulesCount.ts +++ b/packages/crawler/mq/task/getTimeoutSchedulesCount.ts @@ -1,13 +1,11 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; +import { sql } from "@core/db/dbNew"; -export async function getTimeoutSchedulesCount(client: Client) { - const query: string = ` - SELECT COUNT(id) - FROM snapshot_schedule - WHERE status IN ('pending', 'processing') - AND started_at < NOW() - INTERVAL '30 minutes' +export async function getTimeoutSchedulesCount() { + const rows = await sql<{ count: number }[]>` + SELECT COUNT(id) + FROM snapshot_schedule + WHERE status IN ('pending', 'processing') + AND started_at < NOW() - INTERVAL '30 minutes' `; - - const { rows } = await client.queryObject<{ count: number }>(query); return rows[0].count; } diff --git a/packages/crawler/mq/task/getVideoDetails.ts b/packages/crawler/mq/task/getVideoDetails.ts index 40224b0..80a5867 100644 --- a/packages/crawler/mq/task/getVideoDetails.ts +++ b/packages/crawler/mq/task/getVideoDetails.ts @@ -1,13 +1,13 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { getVideoDetails } from "net/getVideoDetails.ts"; import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { ClassifyVideoQueue } from "mq/index.ts"; import { userExistsInBiliUsers, videoExistsInAllData } from "../../db/bilibili_metadata.ts"; -import { HOUR, SECOND } from "@std/datetime"; +import { HOUR, SECOND } from "@core/const/time.ts"; +import type { Psql } from "global.d.ts"; -export async function insertVideoInfo(client: Client, aid: number) { - const videoExists = await videoExistsInAllData(client, aid); +export async function insertVideoInfo(sql: Psql, aid: number) { + const videoExists = await videoExistsInAllData(sql, aid); if (videoExists) { return; } @@ -25,34 +25,37 @@ export async function insertVideoInfo(client: Client, aid: number) { const published_at = formatTimestampToPsql(data.View.pubdate * SECOND + 8 * HOUR); const duration = data.View.duration; const cover = data.View.pic; - await client.queryObject( - `INSERT INTO bilibili_metadata (aid, bvid, description, uid, tags, title, published_at, duration, cover_url) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`, - [aid, bvid, desc, uid, tags, title, published_at, duration, cover], - ); - const userExists = await userExistsInBiliUsers(client, aid); + await sql` + INSERT INTO bilibili_metadata (aid, bvid, description, uid, tags, title, published_at, duration, cover_url) + VALUES (${aid}, ${bvid}, ${desc}, ${uid}, ${tags}, ${title}, ${published_at}, ${duration}, ${cover}) + `; + const userExists = await userExistsInBiliUsers(sql, aid); if (!userExists) { - await client.queryObject( - `INSERT INTO bilibili_user (uid, username, "desc", fans) VALUES ($1, $2, $3, $4)`, - [uid, data.View.owner.name, data.Card.card.sign, data.Card.follower], - ); + await sql` + INSERT INTO bilibili_user (uid, username, "desc", fans) + VALUES (${uid}, ${data.View.owner.name}, ${data.Card.card.sign}, ${data.Card.follower}) + `; } else { - await client.queryObject( - `UPDATE bilibili_user SET fans = $1 WHERE uid = $2`, - [data.Card.follower, uid], - ); + await sql` + UPDATE bilibili_user SET fans = ${data.Card.follower} WHERE uid = ${uid} + `; } const stat = data.View.stat; - const query: string = ` - INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - `; - await client.queryObject( - query, - [aid, stat.view, stat.danmaku, stat.reply, stat.like, stat.coin, stat.share, stat.favorite], - ); + await sql` + INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) + VALUES ( + ${aid}, + ${stat.view}, + ${stat.danmaku}, + ${stat.reply}, + ${stat.like}, + ${stat.coin}, + ${stat.share}, + ${stat.favorite} + ) + ` logger.log(`Inserted video metadata for aid: ${aid}`, "mq"); await ClassifyVideoQueue.add("classifyVideo", { aid }); diff --git a/packages/crawler/mq/task/getVideoStats.ts b/packages/crawler/mq/task/getVideoStats.ts index e09599c..2a7d7ff 100644 --- a/packages/crawler/mq/task/getVideoStats.ts +++ b/packages/crawler/mq/task/getVideoStats.ts @@ -1,6 +1,6 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { getVideoInfo } from "@core/net/getVideoInfo.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; +import type { Psql } from "global.d.ts"; export interface SnapshotNumber { time: number; @@ -25,7 +25,7 @@ export interface SnapshotNumber { * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR` */ export async function insertVideoSnapshot( - client: Client, + sql: Psql, aid: number, task: string, ): Promise { @@ -42,14 +42,10 @@ export async function insertVideoSnapshot( const shares = data.stat.share; const favorites = data.stat.favorite; - const query: string = ` + await sql` INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - `; - await client.queryObject( - query, - [aid, views, danmakus, replies, likes, coins, shares, favorites], - ); + VALUES (${aid}, ${views}, ${danmakus}, ${replies}, ${likes}, ${coins}, ${shares}, ${favorites}) + ` logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot"); diff --git a/packages/crawler/mq/task/queueLatestVideo.ts b/packages/crawler/mq/task/queueLatestVideo.ts index 6327860..af824f7 100644 --- a/packages/crawler/mq/task/queueLatestVideo.ts +++ b/packages/crawler/mq/task/queueLatestVideo.ts @@ -1,13 +1,13 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; import { getLatestVideoAids } from "net/getLatestVideoAids.ts"; -import { videoExistsInAllData } from "../../db/bilibili_metadata.ts"; +import { videoExistsInAllData } from "db/bilibili_metadata.ts"; import { sleep } from "utils/sleep.ts"; -import { SECOND } from "@std/datetime"; -import logger from "log/logger.ts"; +import { SECOND } from "@core/const/time.ts"; +import logger from "@core/log/logger.ts"; import { LatestVideosQueue } from "mq/index.ts"; +import type { Psql } from "global.d.ts"; export async function queueLatestVideos( - client: Client, + sql: Psql, ): Promise { let page = 1; let i = 0; @@ -22,7 +22,7 @@ export async function queueLatestVideos( let allExists = true; let delay = 0; for (const aid of aids) { - const videoExists = await videoExistsInAllData(client, aid); + const videoExists = await videoExistsInAllData(sql, aid); if (videoExists) { continue; } diff --git a/packages/crawler/mq/task/regularSnapshotInterval.ts b/packages/crawler/mq/task/regularSnapshotInterval.ts index 84d4b0b..11871d8 100644 --- a/packages/crawler/mq/task/regularSnapshotInterval.ts +++ b/packages/crawler/mq/task/regularSnapshotInterval.ts @@ -1,13 +1,13 @@ import { findClosestSnapshot, findSnapshotBefore, getLatestSnapshot } from "db/snapshotSchedule.ts"; -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { HOUR } from "@std/datetime"; +import { HOUR } from "@core/const/time.ts"; +import type { Psql } from "global.d.ts"; -export const getRegularSnapshotInterval = async (client: Client, aid: number) => { +export const getRegularSnapshotInterval = async (sql: Psql, aid: number) => { const now = Date.now(); const date = new Date(now - 24 * HOUR); - let oldSnapshot = await findSnapshotBefore(client, aid, date); - if (!oldSnapshot) oldSnapshot = await findClosestSnapshot(client, aid, date); - const latestSnapshot = await getLatestSnapshot(client, aid); + let oldSnapshot = await findSnapshotBefore(sql, aid, date); + if (!oldSnapshot) oldSnapshot = await findClosestSnapshot(sql, aid, date); + const latestSnapshot = await getLatestSnapshot(sql, aid); if (!oldSnapshot || !latestSnapshot) return 0; if (oldSnapshot.created_at === latestSnapshot.created_at) return 0; const hoursDiff = (latestSnapshot.created_at - oldSnapshot.created_at) / HOUR; diff --git a/packages/crawler/mq/task/removeAllTimeoutSchedules.ts b/packages/crawler/mq/task/removeAllTimeoutSchedules.ts index bb8c382..ccca69a 100644 --- a/packages/crawler/mq/task/removeAllTimeoutSchedules.ts +++ b/packages/crawler/mq/task/removeAllTimeoutSchedules.ts @@ -1,16 +1,15 @@ -import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import logger from "log/logger.ts"; +import { sql } from "@core/db/dbNew"; +import logger from "@core/log/logger.ts"; -export async function removeAllTimeoutSchedules(client: Client) { +export async function removeAllTimeoutSchedules() { logger.log( "Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker", ); - const query: string = ` - DELETE FROM snapshot_schedule - WHERE status IN ('pending', 'processing') - AND started_at < NOW() - INTERVAL '30 minutes' - `; - await client.queryObject(query); + return await sql` + DELETE FROM snapshot_schedule + WHERE status IN ('pending', 'processing') + AND started_at < NOW() - INTERVAL '30 minutes' + `; } \ No newline at end of file diff --git a/packages/crawler/net/bulkGetVideoStats.ts b/packages/crawler/net/bulkGetVideoStats.ts index e3b3506..bd586c9 100644 --- a/packages/crawler/net/bulkGetVideoStats.ts +++ b/packages/crawler/net/bulkGetVideoStats.ts @@ -1,6 +1,6 @@ import networkDelegate from "@core/net/delegate.ts"; -import { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d.ts"; -import logger from "log/logger.ts"; +import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d.ts"; +import logger from "@core/log/logger.ts"; /* * Bulk fetch video metadata from bilibili API diff --git a/packages/crawler/net/getLatestVideoAids.ts b/packages/crawler/net/getLatestVideoAids.ts index 675a12d..f466653 100644 --- a/packages/crawler/net/getLatestVideoAids.ts +++ b/packages/crawler/net/getLatestVideoAids.ts @@ -1,5 +1,5 @@ -import { VideoListResponse } from "@core/net/bilibili.d.ts"; -import logger from "log/logger.ts"; +import type { VideoListResponse } from "@core/net/bilibili.d.ts"; +import logger from "@core/log/logger.ts"; import networkDelegate from "@core/net/delegate.ts"; export async function getLatestVideoAids(page: number = 1, pageSize: number = 10): Promise { diff --git a/packages/crawler/net/getVideoDetails.ts b/packages/crawler/net/getVideoDetails.ts index ce15054..13c942e 100644 --- a/packages/crawler/net/getVideoDetails.ts +++ b/packages/crawler/net/getVideoDetails.ts @@ -1,6 +1,6 @@ import networkDelegate from "@core/net/delegate.ts"; -import { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d.ts"; -import logger from "log/logger.ts"; +import type { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d.ts"; +import logger from "@core/log/logger.ts"; export async function getVideoDetails(aid: number): Promise { const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`; diff --git a/packages/crawler/package.json b/packages/crawler/package.json new file mode 100644 index 0000000..de74710 --- /dev/null +++ b/packages/crawler/package.json @@ -0,0 +1,24 @@ +{ + "name": "core", + "scripts": { + "test": "bun --env-file=.env.test run vitest", + "worker:main": "bun run ./src/worker.ts", + "preworker:filter": "bun run ./src/build.ts", + "worker:filter": "bun run ./build/filterWorker.js", + "adder": "bun run ./src/jobAdder.ts", + "bullui": "bun run ./src/bullui.ts", + "all": "bun run concurrently --restart-tries -1 'bun run worker:main' 'bun run adder' 'bun run bullui' 'bun run worker:filter'" + }, + "devDependencies": { + "concurrently": "^9.1.2" + }, + "dependencies": { + "@bull-board/api": "^6.9.5", + "@bull-board/express": "^6.9.5", + "@huggingface/transformers": "^3.5.1", + "bullmq": "^5.52.1", + "express": "^5.1.0", + "ioredis": "^5.6.1", + "onnxruntime-node": "1.19.2" + } +} diff --git a/packages/crawler/src/build.ts b/packages/crawler/src/build.ts new file mode 100644 index 0000000..2d4106b --- /dev/null +++ b/packages/crawler/src/build.ts @@ -0,0 +1,15 @@ +import Bun from "bun"; + +await Bun.build({ + entrypoints: ["./src/filterWorker.ts"], + outdir: "./build", + target: "node" +}); + + +const file = Bun.file("./build/filterWorker.js"); +const code = await file.text(); + +const modifiedCode = code.replaceAll("../bin/napi-v3/", "../node_modules/onnxruntime-node/bin/napi-v3/"); + +await Bun.write("./build/filterWorker.js", modifiedCode); \ No newline at end of file diff --git a/packages/crawler/src/filterWorker.ts b/packages/crawler/src/filterWorker.ts index 7f656f3..7ba2de6 100644 --- a/packages/crawler/src/filterWorker.ts +++ b/packages/crawler/src/filterWorker.ts @@ -1,22 +1,19 @@ import { ConnectionOptions, Job, Worker } from "bullmq"; -import { redis } from "../../core/db/redis.ts"; -import logger from "log/logger.ts"; +import { redis } from "@core/db/redis.ts"; +import logger from "@core/log/logger.ts"; import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts"; import { WorkerError } from "mq/schema.ts"; import { lockManager } from "mq/lockManager.ts"; import Akari from "ml/akari.ts"; -Deno.addSignalListener("SIGINT", async () => { - logger.log("SIGINT Received: Shutting down workers...", "mq"); - await filterWorker.close(true); - Deno.exit(); -}); +const shutdown = async (signal: string) => { + logger.log(`${signal} Received: Shutting down workers...`, "mq"); + await filterWorker.close(true); + process.exit(0); +}; -Deno.addSignalListener("SIGTERM", async () => { - logger.log("SIGTERM Received: Shutting down workers...", "mq"); - await filterWorker.close(true); - Deno.exit(); -}); +process.on('SIGINT', () => shutdown('SIGINT')); +process.on('SIGTERM', () => shutdown('SIGTERM')); await Akari.init(); diff --git a/packages/crawler/src/worker.ts b/packages/crawler/src/worker.ts index 0f0c164..ca644df 100644 --- a/packages/crawler/src/worker.ts +++ b/packages/crawler/src/worker.ts @@ -13,7 +13,7 @@ import { takeBulkSnapshotForVideosWorker, } from "mq/exec/executors.ts"; import { redis } from "@core/db/redis.ts"; -import logger from "log/logger.ts"; +import logger from "@core/log/logger.ts"; import { lockManager } from "mq/lockManager.ts"; import { WorkerError } from "mq/schema.ts"; @@ -29,21 +29,16 @@ const releaseAllLocks = async () => { } }; -Deno.addSignalListener("SIGINT", async () => { - logger.log("SIGINT Received: Shutting down workers...", "mq"); - await releaseAllLocks(); +const shutdown = async (signal: string) => { + logger.log(`${signal} Received: Shutting down workers...`, "mq"); + await releaseAllLocks(); await latestVideoWorker.close(true); await snapshotWorker.close(true); - Deno.exit(); -}); + process.exit(0); +}; -Deno.addSignalListener("SIGTERM", async () => { - logger.log("SIGTERM Received: Shutting down workers...", "mq"); - await releaseAllLocks(); - await latestVideoWorker.close(true); - await snapshotWorker.close(true); - Deno.exit(); -}); +process.on('SIGINT', () => shutdown('SIGINT')); +process.on('SIGTERM', () => shutdown('SIGTERM')); const latestVideoWorker = new Worker( "latestVideos", diff --git a/packages/crawler/test/db/snapshotSchedule.test.ts b/packages/crawler/test/db/snapshotSchedule.test.ts new file mode 100644 index 0000000..b0ff0c1 --- /dev/null +++ b/packages/crawler/test/db/snapshotSchedule.test.ts @@ -0,0 +1,120 @@ +import { expect, test } from "vitest"; +import { sqlTest as sql } from "@core/db/dbNew"; +import { SnapshotScheduleType } from "@core/db/schema"; +import { bulkSetSnapshotStatus } from "db/snapshotSchedule"; + +const mockSnapshotSchedules: SnapshotScheduleType[] = [ + { + id: 1, + aid: 1234567890, + type: "normal", + created_at: "2025-05-04T00:00:00.000Z", + started_at: "2025-05-04T06:00:00.000Z", + finished_at: "2025-05-04T06:03:27.000Z", + status: "completed" + }, + { + id: 2, + aid: 9876543210, + type: "archive", + created_at: "2025-05-03T12:00:00.000Z", + started_at: "2025-05-03T13:00:00.000Z", + finished_at: null, + status: "failed" + }, + { + id: 3, + aid: 1122334455, + type: "milestone", + created_at: "2025-05-01T08:00:00.000Z", + started_at: "2025-05-01T08:12:00.000Z", + finished_at: null, + status: "processing" + } +]; + +const databasePreparationQuery = ` + CREATE SEQUENCE "snapshot_schedule_id_seq" + START WITH 1 + INCREMENT BY 1 + MINVALUE 1 + MAXVALUE 9223372036854775807 + CACHE 1; + + CREATE TABLE "snapshot_schedule"( + "id" bigint DEFAULT nextval('snapshot_schedule_id_seq'::regclass) NOT NULL, + "aid" bigint NOT NULL, + "type" text, + "created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, + "started_at" timestamp with time zone, + "finished_at" timestamp with time zone, + "status" text DEFAULT 'pending'::text NOT NULL + ); + + CREATE INDEX idx_snapshot_schedule_aid ON snapshot_schedule USING btree (aid); + CREATE INDEX idx_snapshot_schedule_started_at ON snapshot_schedule USING btree (started_at); + CREATE INDEX idx_snapshot_schedule_status ON snapshot_schedule USING btree (status); + CREATE INDEX idx_snapshot_schedule_type ON snapshot_schedule USING btree (type); + CREATE UNIQUE INDEX snapshot_schedule_pkey ON snapshot_schedule USING btree (id); +` + +const cleanUpQuery = ` + DROP SEQUENCE IF EXISTS "snapshot_schedule_id_seq" CASCADE; + DROP TABLE IF EXISTS "snapshot_schedule" CASCADE; +` + +async function testMocking() { + await sql.begin(async tx => { + await tx.unsafe(cleanUpQuery).simple(); + await tx.unsafe(databasePreparationQuery).simple(); + + await tx` + INSERT INTO snapshot_schedule + ${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')} + `; + + await tx` + ROLLBACK; + ` + + await tx.unsafe(cleanUpQuery).simple(); + return; + }); +} + +async function testBulkSetSnapshotStatus() { + return await sql.begin(async tx => { + await tx.unsafe(cleanUpQuery).simple(); + await tx.unsafe(databasePreparationQuery).simple(); + + await tx` + INSERT INTO snapshot_schedule + ${sql(mockSnapshotSchedules, 'aid', 'created_at', 'finished_at', 'id', 'started_at', 'status', 'type')} + `; + + const ids = [1, 2, 3]; + + await bulkSetSnapshotStatus(tx, ids, 'pending') + + const rows = tx<{status: string}[]>` + SELECT status FROM snapshot_schedule WHERE id = 1; + `.execute(); + + await tx` + ROLLBACK; + ` + + await tx.unsafe(cleanUpQuery).simple(); + return rows; + }); +} + +test("data mocking works", async () => { + await testMocking(); + expect(() => {}).not.toThrowError(); +}); + +test("bulkSetSnapshotStatus core logic works smoothly", async () => { + const rows = await testBulkSetSnapshotStatus(); + expect(rows.every(item => item.status === 'pending')).toBe(true); +}); diff --git a/packages/crawler/tsconfig.json b/packages/crawler/tsconfig.json new file mode 100644 index 0000000..b344664 --- /dev/null +++ b/packages/crawler/tsconfig.json @@ -0,0 +1,18 @@ +{ + "include": ["**/*.ts"], + "compilerOptions": { + "baseUrl": ".", + "target": "ESNext", + "module": "ESNext", + "useDefineForClassFields": true, + "moduleResolution": "node", + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "skipLibCheck": true, + "paths": { + "@core/*": ["../core/*"] + }, + "allowSyntheticDefaultImports": true, + "allowImportingTsExtensions": true, + "noEmit": true + } +} diff --git a/packages/crawler/vitest.config.ts b/packages/crawler/vitest.config.ts new file mode 100644 index 0000000..fb5072b --- /dev/null +++ b/packages/crawler/vitest.config.ts @@ -0,0 +1,6 @@ +import { defineConfig } from "vitest/config"; +import tsconfigPaths from 'vite-tsconfig-paths' + +export default defineConfig({ + plugins: [tsconfigPaths()] +});