ref: use bun's redis in crawler, deployment via PM2
This commit is contained in:
parent
57992069ec
commit
1a79e679bb
@ -1,6 +1,6 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="BunSettings">
|
||||
<option name="bunPath" value="$USER_HOME$/.bun/bin/bun" />
|
||||
<option name="bunPath" value="/opt/homebrew/bin/bun" />
|
||||
</component>
|
||||
</project>
|
||||
@ -1,3 +0,0 @@
|
||||
<component name="DependencyValidationManager">
|
||||
<scope name="Astro" pattern="file:*.astro" />
|
||||
</component>
|
||||
87
bun.lock
87
bun.lock
@ -8,7 +8,7 @@
|
||||
"postgres": "^3.4.7",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.2.22",
|
||||
"@types/bun": "^1.3.1",
|
||||
"prettier": "^3.6.2",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"vitest": "^3.2.4",
|
||||
@ -40,6 +40,11 @@
|
||||
"name": "@cvsa/core",
|
||||
"version": "0.0.10",
|
||||
"dependencies": {
|
||||
"@alicloud/credentials": "^2.4.4",
|
||||
"@alicloud/darabonba-stream": "^0.0.2",
|
||||
"@alicloud/fc20230330": "^4.6.2",
|
||||
"@alicloud/openapi-client": "^0.4.15",
|
||||
"@alicloud/tea-util": "^1.4.10",
|
||||
"@koshnic/ratelimit": "^1.0.3",
|
||||
"@types/luxon": "^3.7.1",
|
||||
"chalk": "^5.4.1",
|
||||
@ -223,6 +228,40 @@
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@alicloud/credentials": ["@alicloud/credentials@2.4.4", "", { "dependencies": { "@alicloud/tea-typescript": "^1.8.0", "httpx": "^2.3.3", "ini": "^1.3.5", "kitx": "^2.0.0" } }, "sha512-/eRAGSKcniLIFQ1UCpDhB/IrHUZisQ1sc65ws/c2avxUMpXwH1rWAohb76SVAUJhiF4mwvLzLJM1Mn1XL4Xe/Q=="],
|
||||
|
||||
"@alicloud/darabonba-array": ["@alicloud/darabonba-array@0.1.2", "", { "dependencies": { "@alicloud/tea-typescript": "^1.7.1" } }, "sha512-ZPuQ+bJyjrd8XVVm55kl+ypk7OQoi1ZH/DiToaAEQaGvgEjrTcvQkg71//vUX/6cvbLIF5piQDvhrLb+lUEIPQ=="],
|
||||
|
||||
"@alicloud/darabonba-encode-util": ["@alicloud/darabonba-encode-util@0.0.2", "", { "dependencies": { "moment": "^2.29.1" } }, "sha512-mlsNctkeqmR0RtgE1Rngyeadi5snLOAHBCWEtYf68d7tyKskosXDTNeZ6VCD/UfrUu4N51ItO8zlpfXiOgeg3A=="],
|
||||
|
||||
"@alicloud/darabonba-map": ["@alicloud/darabonba-map@0.0.1", "", { "dependencies": { "@alicloud/tea-typescript": "^1.7.1" } }, "sha512-2ep+G3YDvuI+dRYVlmER1LVUQDhf9kEItmVB/bbEu1pgKzelcocCwAc79XZQjTcQGFgjDycf3vH87WLDGLFMlw=="],
|
||||
|
||||
"@alicloud/darabonba-signature-util": ["@alicloud/darabonba-signature-util@0.0.4", "", { "dependencies": { "@alicloud/darabonba-encode-util": "^0.0.1" } }, "sha512-I1TtwtAnzLamgqnAaOkN0IGjwkiti//0a7/auyVThdqiC/3kyafSAn6znysWOmzub4mrzac2WiqblZKFcN5NWg=="],
|
||||
|
||||
"@alicloud/darabonba-stream": ["@alicloud/darabonba-stream@0.0.2", "", { "dependencies": { "@alicloud/tea-typescript": "^1.7.1" } }, "sha512-eYViOqHa42nlJRwS1mjEs8hYIvJ0SaoHR7IDA2xyKjwdtyi6LJZbbEbaW9O0CjNPghm8+HB20VxAkcBNzVL0kg=="],
|
||||
|
||||
"@alicloud/darabonba-string": ["@alicloud/darabonba-string@1.0.3", "", { "dependencies": { "@alicloud/tea-typescript": "^1.5.1" } }, "sha512-NyWwrU8cAIesWk3uHL1Q7pTDTqLkCI/0PmJXC4/4A0MFNAZ9Ouq0iFBsRqvfyUujSSM+WhYLuTfakQXiVLkTMA=="],
|
||||
|
||||
"@alicloud/endpoint-util": ["@alicloud/endpoint-util@0.0.1", "", { "dependencies": { "@alicloud/tea-typescript": "^1.5.1", "kitx": "^2.0.0" } }, "sha512-+pH7/KEXup84cHzIL6UJAaPqETvln4yXlD9JzlrqioyCSaWxbug5FUobsiI6fuUOpw5WwoB3fWAtGbFnJ1K3Yg=="],
|
||||
|
||||
"@alicloud/fc20230330": ["@alicloud/fc20230330@4.6.2", "", { "dependencies": { "@alicloud/openapi-core": "^1.0.0", "@darabonba/typescript": "^1.0.0" } }, "sha512-wrh8JK88CrM0t00GXW/hWkVBoLX2ReOM7nf8Hem6gaWeoB9SE9XX5j/eV7lBpmb7AwzCj8Tc5nXBNpByZhSGlw=="],
|
||||
|
||||
"@alicloud/gateway-pop": ["@alicloud/gateway-pop@0.0.6", "", { "dependencies": { "@alicloud/credentials": "^2", "@alicloud/darabonba-array": "^0.1.0", "@alicloud/darabonba-encode-util": "^0.0.2", "@alicloud/darabonba-map": "^0.0.1", "@alicloud/darabonba-signature-util": "^0.0.4", "@alicloud/darabonba-string": "^1.0.2", "@alicloud/endpoint-util": "^0.0.1", "@alicloud/gateway-spi": "^0.0.8", "@alicloud/openapi-util": "^0.3.2", "@alicloud/tea-typescript": "^1.7.1", "@alicloud/tea-util": "^1.4.8" } }, "sha512-KF4I+JvfYuLKc3fWeWYIZ7lOVJ9jRW0sQXdXidZn1DKZ978ncfGf7i0LBfONGk4OxvNb/HD3/0yYhkgZgPbKtA=="],
|
||||
|
||||
"@alicloud/gateway-spi": ["@alicloud/gateway-spi@0.0.8", "", { "dependencies": { "@alicloud/credentials": "^2", "@alicloud/tea-typescript": "^1.7.1" } }, "sha512-KM7fu5asjxZPmrz9sJGHJeSU+cNQNOxW+SFmgmAIrITui5hXL2LB+KNRuzWmlwPjnuA2X3/keq9h6++S9jcV5g=="],
|
||||
|
||||
"@alicloud/openapi-client": ["@alicloud/openapi-client@0.4.15", "", { "dependencies": { "@alicloud/credentials": "^2.4.2", "@alicloud/gateway-spi": "^0.0.8", "@alicloud/openapi-util": "^0.3.2", "@alicloud/tea-typescript": "^1.7.1", "@alicloud/tea-util": "1.4.9", "@alicloud/tea-xml": "0.0.3" } }, "sha512-4VE0/k5ZdQbAhOSTqniVhuX1k5DUeUMZv74degn3wIWjLY6Bq+hxjaGsaHYlLZ2gA5wUrs8NcI5TE+lIQS3iiA=="],
|
||||
|
||||
"@alicloud/openapi-core": ["@alicloud/openapi-core@1.0.5", "", { "dependencies": { "@alicloud/credentials": "latest", "@alicloud/gateway-pop": "0.0.6", "@alicloud/gateway-spi": "^0.0.8", "@darabonba/typescript": "^1.0.2" } }, "sha512-ed4EKyqHjb9zwrXUs6IRthha/pRn3OUoOcUKuhYu4tllp0RpidA+JYswsweN6sh26H0WIs/LB6nzJEOvh1d3fg=="],
|
||||
|
||||
"@alicloud/openapi-util": ["@alicloud/openapi-util@0.3.2", "", { "dependencies": { "@alicloud/tea-typescript": "^1.7.1", "@alicloud/tea-util": "^1.3.0", "kitx": "^2.1.0", "sm3": "^1.0.3" } }, "sha512-EC2JvxdcOgMlBAEG0+joOh2IB1um8CPz9EdYuRfTfd1uP8Yc9D8QRUWVGjP6scnj6fWSOaHFlit9H6PrJSyFow=="],
|
||||
|
||||
"@alicloud/tea-typescript": ["@alicloud/tea-typescript@1.8.0", "", { "dependencies": { "@types/node": "^12.0.2", "httpx": "^2.2.6" } }, "sha512-CWXWaquauJf0sW30mgJRVu9aaXyBth5uMBCUc+5vKTK1zlgf3hIqRUjJZbjlwHwQ5y9anwcu18r48nOZb7l2QQ=="],
|
||||
|
||||
"@alicloud/tea-util": ["@alicloud/tea-util@1.4.10", "", { "dependencies": { "@alicloud/tea-typescript": "^1.5.1", "@darabonba/typescript": "^1.0.0", "kitx": "^2.0.0" } }, "sha512-VEsXWP2dlJLvsY2THj+sH++zwxQRz3Y5BQ8EkfnFems36RkngQKYOLsoto5nR6ej1Gf6I+0IOgBXrkRdpNCQ1g=="],
|
||||
|
||||
"@alicloud/tea-xml": ["@alicloud/tea-xml@0.0.3", "", { "dependencies": { "@alicloud/tea-typescript": "^1", "@types/xml2js": "^0.4.5", "xml2js": "^0.6.0" } }, "sha512-+/9GliugjrLglsXVrd1D80EqqKgGpyA0eQ6+1ZdUOYCaRguaSwz44trX3PaxPu/HhIPJg9PsGQQ3cSLXWZjbAA=="],
|
||||
|
||||
"@alikia/dark-theme-hook": ["@alikia/dark-theme-hook@1.0.2", "", { "dependencies": { "react": "^18.3.1" } }, "sha512-OloatZRefHB7Ey3zjhfsKFZoHbfzewPfOeEwA7q9zDXViNKGyTA4CiCF3US5vzqfhpR16wpYcPSmpVabKI3MYg=="],
|
||||
|
||||
"@alloc/quick-lru": ["@alloc/quick-lru@5.2.0", "", {}, "sha512-UrcABB+4bUrFABwbluTIBErXwvbsU/V7TZWfmbgJfbkwiBuziS9gxdODUyuiecfdGQ85jglMW6juS3+z5TsKLw=="],
|
||||
@ -353,6 +392,8 @@
|
||||
|
||||
"@dabh/diagnostics": ["@dabh/diagnostics@2.0.3", "", { "dependencies": { "colorspace": "1.1.x", "enabled": "2.0.x", "kuler": "^2.0.0" } }, "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA=="],
|
||||
|
||||
"@darabonba/typescript": ["@darabonba/typescript@1.0.3", "", { "dependencies": { "@alicloud/tea-typescript": "^1.5.1", "httpx": "^2.3.2", "lodash": "^4.17.21", "moment": "^2.30.1", "moment-timezone": "^0.5.45", "xml2js": "^0.6.2" } }, "sha512-/y2y6wf5TsxD7pCPIm0OvTC+5qV0Tk7HQYxwpIuWRLXQLB0CRDvr6qk4bR6rTLO/JglJa8z2uCGZsaLYpQNqFQ=="],
|
||||
|
||||
"@deno/shim-deno": ["@deno/shim-deno@0.19.2", "", { "dependencies": { "@deno/shim-deno-test": "^0.5.0", "which": "^4.0.0" } }, "sha512-q3VTHl44ad8T2Tw2SpeAvghdGOjlnLPDNO2cpOxwMrBE/PVas6geWpbpIgrM+czOCH0yejp0yi8OaTuB+NU40Q=="],
|
||||
|
||||
"@deno/shim-deno-test": ["@deno/shim-deno-test@0.5.0", "", {}, "sha512-4nMhecpGlPi0cSzT67L+Tm+GOJqvuk8gqHBziqcUQOarnuIax1z96/gJHCSIz2Z0zhxE6Rzwb3IZXPtFh51j+w=="],
|
||||
@ -889,7 +930,7 @@
|
||||
|
||||
"@types/braces": ["@types/braces@3.0.5", "", {}, "sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.2.22", "", { "dependencies": { "bun-types": "1.2.22" } }, "sha512-5A/KrKos2ZcN0c6ljRSOa1fYIyCKhZfIVYeuyb4snnvomnpFqC0tTsEkdqNxbAgExV384OETQ//WAjl3XbYqQA=="],
|
||||
"@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
|
||||
|
||||
"@types/chai": ["@types/chai@5.2.2", "", { "dependencies": { "@types/deep-eql": "*" } }, "sha512-8kB30R7Hwqf40JPiKhVzodJs2Qc1ZJ5zuT3uzw5Hq/dhNCl3G3l83jfpdI1e20BP348+fV7VIL/+FxaXkqBmWg=="],
|
||||
|
||||
@ -965,6 +1006,8 @@
|
||||
|
||||
"@types/use-sync-external-store": ["@types/use-sync-external-store@0.0.6", "", {}, "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg=="],
|
||||
|
||||
"@types/xml2js": ["@types/xml2js@0.4.14", "", { "dependencies": { "@types/node": "*" } }, "sha512-4YnrRemBShWRO2QjvUin8ESA41rH+9nQGLUGZV/1IDhi3SL9OhdpNC/MrulTWuptXKwhx/aDxE7toV0f/ypIXQ=="],
|
||||
|
||||
"@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.45.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.10.0", "@typescript-eslint/scope-manager": "8.45.0", "@typescript-eslint/type-utils": "8.45.0", "@typescript-eslint/utils": "8.45.0", "@typescript-eslint/visitor-keys": "8.45.0", "graphemer": "^1.4.0", "ignore": "^7.0.0", "natural-compare": "^1.4.0", "ts-api-utils": "^2.1.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.45.0", "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-HC3y9CVuevvWCl/oyZuI47dOeDF9ztdMEfMH8/DW/Mhwa9cCLnK1oD7JoTVGW/u7kFzNZUKUoyJEqkaJh5y3Wg=="],
|
||||
|
||||
"@typescript-eslint/parser": ["@typescript-eslint/parser@8.45.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.45.0", "@typescript-eslint/types": "8.45.0", "@typescript-eslint/typescript-estree": "8.45.0", "@typescript-eslint/visitor-keys": "8.45.0", "debug": "^4.3.4" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0", "typescript": ">=4.8.4 <6.0.0" } }, "sha512-TGf22kon8KW+DeKaUmOibKWktRY8b2NSAZNdtWh798COm1NWx8+xJ6iFBtk3IvLdv6+LGLJLRlyhrhEDZWargQ=="],
|
||||
@ -1717,6 +1760,8 @@
|
||||
|
||||
"https-proxy-agent": ["https-proxy-agent@7.0.6", "", { "dependencies": { "agent-base": "^7.1.2", "debug": "4" } }, "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw=="],
|
||||
|
||||
"httpx": ["httpx@2.3.3", "", { "dependencies": { "@types/node": "^20", "debug": "^4.1.1" } }, "sha512-k1qv94u1b6e+XKCxVbLgYlOypVP9MPGpnN5G/vxFf6tDO4V3xpz3d6FUOY/s8NtPgaq5RBVVgSB+7IHpVxMYzw=="],
|
||||
|
||||
"httpxy": ["httpxy@0.1.7", "", {}, "sha512-pXNx8gnANKAndgga5ahefxc++tJvNL87CXoRwxn1cJE2ZkWEojF3tNfQIEhZX/vfpt+wzeAzpUI4qkediX1MLQ=="],
|
||||
|
||||
"human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="],
|
||||
@ -1739,6 +1784,8 @@
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"ini": ["ini@1.3.8", "", {}, "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="],
|
||||
|
||||
"internmap": ["internmap@2.0.3", "", {}, "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg=="],
|
||||
|
||||
"ioredis": ["ioredis@5.7.0", "", { "dependencies": { "@ioredis/commands": "^1.3.0", "cluster-key-slot": "^1.1.0", "debug": "^4.3.4", "denque": "^2.1.0", "lodash.defaults": "^4.2.0", "lodash.isarguments": "^3.1.0", "redis-errors": "^1.2.0", "redis-parser": "^3.0.0", "standard-as-callback": "^2.1.0" } }, "sha512-NUcA93i1lukyXU+riqEyPtSEkyFq8tX90uL659J+qpCZ3rEdViB/APC58oAhIh3+bJln2hzdlZbBZsGNrlsR8g=="],
|
||||
@ -1821,6 +1868,8 @@
|
||||
|
||||
"keyv": ["keyv@4.5.4", "", { "dependencies": { "json-buffer": "3.0.1" } }, "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw=="],
|
||||
|
||||
"kitx": ["kitx@2.2.0", "", { "dependencies": { "@types/node": "^22.5.4" } }, "sha512-tBMwe6AALTBQJb0woQDD40734NKzb0Kzi3k7wQj9ar3AbP9oqhoVrdXPh7rk2r00/glIgd0YbToIUJsnxWMiIg=="],
|
||||
|
||||
"kleur": ["kleur@4.1.5", "", {}, "sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ=="],
|
||||
|
||||
"klona": ["klona@2.0.6", "", {}, "sha512-dhG34DXATL5hSxJbIexCft8FChFXtmskoZYnoPWjXQuebWYCNkVeV3KkGegCK9CP1oswI/vQibS2GY7Em/sJJA=="],
|
||||
@ -2025,6 +2074,10 @@
|
||||
|
||||
"mocha": ["mocha@10.8.2", "", { "dependencies": { "ansi-colors": "^4.1.3", "browser-stdout": "^1.3.1", "chokidar": "^3.5.3", "debug": "^4.3.5", "diff": "^5.2.0", "escape-string-regexp": "^4.0.0", "find-up": "^5.0.0", "glob": "^8.1.0", "he": "^1.2.0", "js-yaml": "^4.1.0", "log-symbols": "^4.1.0", "minimatch": "^5.1.6", "ms": "^2.1.3", "serialize-javascript": "^6.0.2", "strip-json-comments": "^3.1.1", "supports-color": "^8.1.1", "workerpool": "^6.5.1", "yargs": "^16.2.0", "yargs-parser": "^20.2.9", "yargs-unparser": "^2.0.0" }, "bin": { "mocha": "bin/mocha.js", "_mocha": "bin/_mocha" } }, "sha512-VZlYo/WE8t1tstuRmqgeyBgCbJc/lEdopaa+axcKzTBJ+UIdlAB9XnmvTCAH4pwR4ElNInaedhEBmZD8iCSVEg=="],
|
||||
|
||||
"moment": ["moment@2.30.1", "", {}, "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how=="],
|
||||
|
||||
"moment-timezone": ["moment-timezone@0.5.48", "", { "dependencies": { "moment": "^2.29.4" } }, "sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw=="],
|
||||
|
||||
"morgan": ["morgan@1.10.1", "", { "dependencies": { "basic-auth": "~2.0.1", "debug": "2.6.9", "depd": "~2.0.0", "on-finished": "~2.3.0", "on-headers": "~1.1.0" } }, "sha512-223dMRJtI/l25dJKWpgij2cMtywuG/WiUKXdvwfbhGKBhy1puASqXwFzmWZ7+K73vUPoR7SS2Qz2cI/g9MKw0A=="],
|
||||
|
||||
"motion": ["motion@12.23.22", "", { "dependencies": { "framer-motion": "^12.23.22", "tslib": "^2.4.0" }, "peerDependencies": { "@emotion/is-prop-valid": "*", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" }, "optionalPeers": ["@emotion/is-prop-valid", "react", "react-dom"] }, "sha512-iSq6X9vLHbeYwmHvhK//+U74ROaPnZmBuy60XZzqNl0QtZkWfoZyMDHYnpKuWFv0sNMqHgED8aCXk94LCoQPGg=="],
|
||||
@ -2387,6 +2440,8 @@
|
||||
|
||||
"safer-buffer": ["safer-buffer@2.1.2", "", {}, "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="],
|
||||
|
||||
"sax": ["sax@1.4.1", "", {}, "sha512-+aWOz7yVScEGoKNd4PA10LZ8sk0A/z5+nXQG5giUO5rprX9jgYsTdov9qCchZiPIZezbZH+jRut8nPodFAX4Jg=="],
|
||||
|
||||
"scheduler": ["scheduler@0.26.0", "", {}, "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA=="],
|
||||
|
||||
"scule": ["scule@1.3.0", "", {}, "sha512-6FtHJEvt+pVMIB9IBY+IcCJ6Z5f1iQnytgyfKMhDKgmzYG+TeH/wx1y3l27rshSbLiSanrR9ffZDrEsmjlQF2g=="],
|
||||
@ -2445,6 +2500,8 @@
|
||||
|
||||
"slash": ["slash@3.0.0", "", {}, "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q=="],
|
||||
|
||||
"sm3": ["sm3@1.0.3", "", {}, "sha512-KyFkIfr8QBlFG3uc3NaljaXdYcsbRy1KrSfc4tsQV8jW68jAktGeOcifu530Vx/5LC+PULHT0Rv8LiI8Gw+c1g=="],
|
||||
|
||||
"smob": ["smob@1.5.0", "", {}, "sha512-g6T+p7QO8npa+/hNx9ohv1E5pVCmWrVCUzUXJyLdMmftX6ER0oiWY/w9knEonLpnOp6b6FenKnMfR8gqwWdwig=="],
|
||||
|
||||
"smol-toml": ["smol-toml@1.4.2", "", {}, "sha512-rInDH6lCNiEyn3+hH8KVGFdbjc099j47+OSgbMrfDYX1CmXLfdKd7qi6IfcWj2wFxvSVkuI46M+wPGYfEOEj6g=="],
|
||||
@ -2767,6 +2824,10 @@
|
||||
|
||||
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
|
||||
|
||||
"xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="],
|
||||
|
||||
"xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="],
|
||||
|
||||
"xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="],
|
||||
|
||||
"xxhash-wasm": ["xxhash-wasm@1.1.0", "", {}, "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA=="],
|
||||
@ -2807,6 +2868,12 @@
|
||||
|
||||
"zwitch": ["zwitch@2.0.4", "", {}, "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A=="],
|
||||
|
||||
"@alicloud/darabonba-signature-util/@alicloud/darabonba-encode-util": ["@alicloud/darabonba-encode-util@0.0.1", "", { "dependencies": { "@alicloud/tea-typescript": "^1.7.1", "moment": "^2.29.1" } }, "sha512-Sl5vCRVAYMqwmvXpJLM9hYoCHOMsQlGxaWSGhGWulpKk/NaUBArtoO1B0yHruJf1C5uHhEJIaylYcM48icFHgw=="],
|
||||
|
||||
"@alicloud/openapi-client/@alicloud/tea-util": ["@alicloud/tea-util@1.4.9", "", { "dependencies": { "@alicloud/tea-typescript": "^1.5.1", "kitx": "^2.0.0" } }, "sha512-S0wz76rGtoPKskQtRTGqeuqBHFj8BqUn0Vh+glXKun2/9UpaaaWmuJwcmtImk6bJZfLYEShDF/kxDmDJoNYiTw=="],
|
||||
|
||||
"@alicloud/tea-typescript/@types/node": ["@types/node@12.20.55", "", {}, "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ=="],
|
||||
|
||||
"@alikia/dark-theme-hook/react": ["react@18.3.1", "", { "dependencies": { "loose-envify": "^1.1.0" } }, "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ=="],
|
||||
|
||||
"@antfu/install-pkg/tinyexec": ["tinyexec@1.0.1", "", {}, "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw=="],
|
||||
@ -2825,6 +2892,8 @@
|
||||
|
||||
"@cloudflare/kv-asset-handler/mime": ["mime@3.0.0", "", { "bin": { "mime": "cli.js" } }, "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A=="],
|
||||
|
||||
"@cvsa/backend/@types/bun": ["@types/bun@1.2.22", "", { "dependencies": { "bun-types": "1.2.22" } }, "sha512-5A/KrKos2ZcN0c6ljRSOa1fYIyCKhZfIVYeuyb4snnvomnpFqC0tTsEkdqNxbAgExV384OETQ//WAjl3XbYqQA=="],
|
||||
|
||||
"@deno/shim-deno/which": ["which@4.0.0", "", { "dependencies": { "isexe": "^3.1.1" }, "bin": { "node-which": "bin/which.js" } }, "sha512-GlaYyEb07DPxYCKhKzplCWBJtvxZcZMrL+4UkrTSJHHPyZU4mYYTv3qaOe77H7EODLSSopAUFAc6W8U4yqvscg=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild": ["esbuild@0.18.20", "", { "optionalDependencies": { "@esbuild/android-arm": "0.18.20", "@esbuild/android-arm64": "0.18.20", "@esbuild/android-x64": "0.18.20", "@esbuild/darwin-arm64": "0.18.20", "@esbuild/darwin-x64": "0.18.20", "@esbuild/freebsd-arm64": "0.18.20", "@esbuild/freebsd-x64": "0.18.20", "@esbuild/linux-arm": "0.18.20", "@esbuild/linux-arm64": "0.18.20", "@esbuild/linux-ia32": "0.18.20", "@esbuild/linux-loong64": "0.18.20", "@esbuild/linux-mips64el": "0.18.20", "@esbuild/linux-ppc64": "0.18.20", "@esbuild/linux-riscv64": "0.18.20", "@esbuild/linux-s390x": "0.18.20", "@esbuild/linux-x64": "0.18.20", "@esbuild/netbsd-x64": "0.18.20", "@esbuild/openbsd-x64": "0.18.20", "@esbuild/sunos-x64": "0.18.20", "@esbuild/win32-arm64": "0.18.20", "@esbuild/win32-ia32": "0.18.20", "@esbuild/win32-x64": "0.18.20" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA=="],
|
||||
@ -2903,10 +2972,12 @@
|
||||
|
||||
"@tanstack/server-functions-plugin/@babel/code-frame": ["@babel/code-frame@7.26.2", "", { "dependencies": { "@babel/helper-validator-identifier": "^7.25.9", "js-tokens": "^4.0.0", "picocolors": "^1.0.0" } }, "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ=="],
|
||||
|
||||
"@types/bun/bun-types": ["bun-types@1.2.22", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-hwaAu8tct/Zn6Zft4U9BsZcXkYomzpHJX28ofvx7k0Zz2HNz54n1n+tDgxoWFGB4PcFvJXJQloPhaV2eP3Q6EA=="],
|
||||
"@types/bun/bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
|
||||
|
||||
"@types/d3-scale/@types/d3-time": ["@types/d3-time@3.0.4", "", {}, "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g=="],
|
||||
|
||||
"@types/xml2js/@types/node": ["@types/node@24.6.2", "", { "dependencies": { "undici-types": "~7.13.0" } }, "sha512-d2L25Y4j+W3ZlNAeMKcy7yDsK425ibcAOO2t7aPTz6gNMH0z2GThtwENCDc0d/Pw9wgyRqE5Px1wkV7naz8ang=="],
|
||||
|
||||
"@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="],
|
||||
|
||||
"@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="],
|
||||
@ -3013,6 +3084,8 @@
|
||||
|
||||
"http-proxy/eventemitter3": ["eventemitter3@4.0.7", "", {}, "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw=="],
|
||||
|
||||
"kitx/@types/node": ["@types/node@22.18.12", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-BICHQ67iqxQGFSzfCFTT7MRQ5XcBjG5aeKh5Ok38UBbPe5fxTyE+aHFxwVrGyr8GNlqFMLKD1D3P2K/1ks8tog=="],
|
||||
|
||||
"lazystream/readable-stream": ["readable-stream@2.3.8", "", { "dependencies": { "core-util-is": "~1.0.0", "inherits": "~2.0.3", "isarray": "~1.0.0", "process-nextick-args": "~2.0.0", "safe-buffer": "~5.1.1", "string_decoder": "~1.1.1", "util-deprecate": "~1.0.1" } }, "sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA=="],
|
||||
|
||||
"listhen/@parcel/watcher-wasm": ["@parcel/watcher-wasm@2.5.1", "", { "dependencies": { "is-glob": "^4.0.3", "micromatch": "^4.0.5", "napi-wasm": "^1.1.0" } }, "sha512-RJxlQQLkaMMIuWRozy+z2vEqbaQlCuaCgVZIUCzQLYggY22LZbP5Y1+ia+FD724Ids9e+XIyOLXLrLgQSHIthw=="],
|
||||
@ -3135,6 +3208,8 @@
|
||||
|
||||
"@babel/helper-compilation-targets/lru-cache/yallist": ["yallist@3.1.1", "", {}, "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="],
|
||||
|
||||
"@cvsa/backend/@types/bun/bun-types": ["bun-types@1.2.22", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-hwaAu8tct/Zn6Zft4U9BsZcXkYomzpHJX28ofvx7k0Zz2HNz54n1n+tDgxoWFGB4PcFvJXJQloPhaV2eP3Q6EA=="],
|
||||
|
||||
"@deno/shim-deno/which/isexe": ["isexe@3.1.1", "", {}, "sha512-LpB/54B+/2J5hqQ7imZHfdU31OlgQqx7ZicVlkm9kzg9/w8GKLEcFfJl/t7DCEDueOyBAD6zCCwTO6Fzs0NoEQ=="],
|
||||
|
||||
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="],
|
||||
@ -3247,6 +3322,10 @@
|
||||
|
||||
"@tanstack/server-functions-plugin/@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="],
|
||||
|
||||
"@types/bun/bun-types/@types/node": ["@types/node@24.6.2", "", { "dependencies": { "undici-types": "~7.13.0" } }, "sha512-d2L25Y4j+W3ZlNAeMKcy7yDsK425ibcAOO2t7aPTz6gNMH0z2GThtwENCDc0d/Pw9wgyRqE5Px1wkV7naz8ang=="],
|
||||
|
||||
"@types/xml2js/@types/node/undici-types": ["undici-types@7.13.0", "", {}, "sha512-Ov2Rr9Sx+fRgagJ5AX0qvItZG/JKKoBRAVITs1zk7IqZGTJUwgUr7qoYBpWwakpWilTZFM98rG/AFRocu10iIQ=="],
|
||||
|
||||
"@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="],
|
||||
|
||||
"@unocss/cli/chokidar/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="],
|
||||
@ -3359,6 +3438,8 @@
|
||||
|
||||
"@solidjs/start/shiki/@shikijs/engine-javascript/oniguruma-to-es": ["oniguruma-to-es@2.3.0", "", { "dependencies": { "emoji-regex-xs": "^1.0.0", "regex": "^5.1.1", "regex-recursion": "^5.1.1" } }, "sha512-bwALDxriqfKGfUufKGGepCzu9x7nJQuoRoAFp4AnwehhC2crqrDIAP/uN2qdlsAvSMpeRC3+Yzhqc7hLmle5+g=="],
|
||||
|
||||
"@types/bun/bun-types/@types/node/undici-types": ["undici-types@7.13.0", "", {}, "sha512-Ov2Rr9Sx+fRgagJ5AX0qvItZG/JKKoBRAVITs1zk7IqZGTJUwgUr7qoYBpWwakpWilTZFM98rG/AFRocu10iIQ=="],
|
||||
|
||||
"@unocss/cli/chokidar/readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
|
||||
|
||||
"@unocss/vite/chokidar/readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
|
||||
|
||||
17
ecosystem.config.js
Normal file
17
ecosystem.config.js
Normal file
@ -0,0 +1,17 @@
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: 'crawler-worker',
|
||||
script: 'src/worker.ts',
|
||||
cwd: './packages/api',
|
||||
interpreter: 'bun',
|
||||
instances: 1,
|
||||
autorestart: true,
|
||||
watch: false,
|
||||
max_memory_restart: '1G',
|
||||
env: {
|
||||
PATH: `${process.env.HOME}/.bun/bin:${process.env.PATH}`, // Add "~/.bun/bin/bun" to PATH
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
@ -18,7 +18,7 @@
|
||||
"postgres": "^3.4.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.2.22",
|
||||
"@types/bun": "^1.3.1",
|
||||
"prettier": "^3.6.2",
|
||||
"vite-tsconfig-paths": "^5.1.4",
|
||||
"vitest": "^3.2.4",
|
||||
|
||||
@ -1,14 +1,13 @@
|
||||
import { Redis } from "ioredis";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis, RedisClient } from "bun";
|
||||
|
||||
class LockManager {
|
||||
private redis: Redis;
|
||||
private redis: RedisClient;
|
||||
|
||||
/*
|
||||
* Create a new LockManager
|
||||
* @param redisClient The Redis client used to store the lock data
|
||||
*/
|
||||
constructor(redisClient: Redis) {
|
||||
constructor(redisClient: RedisClient) {
|
||||
this.redis = redisClient;
|
||||
}
|
||||
|
||||
@ -49,8 +48,7 @@ class LockManager {
|
||||
*/
|
||||
async isLocked(id: string): Promise<boolean> {
|
||||
const key = `cvsa:lock:${id}`;
|
||||
const result = await this.redis.exists(key);
|
||||
return result === 1;
|
||||
return await this.redis.exists(key);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import { RateLimiter as Limiter } from "@koshnic/ratelimit";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis } from "bun";
|
||||
import Redis from "ioredis";
|
||||
|
||||
export interface RateLimiterConfig {
|
||||
duration: number;
|
||||
@ -8,6 +9,7 @@ export interface RateLimiterConfig {
|
||||
|
||||
export class RateLimiterError extends Error {
|
||||
public code: string;
|
||||
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "RateLimiterError";
|
||||
@ -28,7 +30,7 @@ export class MultipleRateLimiter {
|
||||
*/
|
||||
constructor(name: string, configs: RateLimiterConfig[]) {
|
||||
this.configs = configs;
|
||||
this.limiter = new Limiter(redis);
|
||||
this.limiter = new Limiter(redis as unknown as Redis);
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
|
||||
@ -1,40 +1,21 @@
|
||||
import logger from "@core/log";
|
||||
import { MultipleRateLimiter, RateLimiterError, type RateLimiterConfig } from "@core/mq/multipleRateLimiter";
|
||||
import {
|
||||
MultipleRateLimiter,
|
||||
RateLimiterError,
|
||||
type RateLimiterConfig
|
||||
} from "@core/mq/multipleRateLimiter";
|
||||
import { ReplyError } from "ioredis";
|
||||
import { SECOND } from "@core/lib";
|
||||
import { spawn, SpawnOptions } from "child_process";
|
||||
import FC20230330, * as $FC20230330 from "@alicloud/fc20230330";
|
||||
import Credential from "@alicloud/credentials";
|
||||
import * as OpenApi from "@alicloud/openapi-client";
|
||||
import Stream from "@alicloud/darabonba-stream";
|
||||
import * as Util from "@alicloud/tea-util";
|
||||
import { Readable } from "stream";
|
||||
|
||||
export function spawnPromise(
|
||||
command: string,
|
||||
args: string[] = [],
|
||||
options?: SpawnOptions
|
||||
): Promise<{ stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, options);
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
|
||||
child.stdout?.on("data", (data) => {
|
||||
stdout += data;
|
||||
});
|
||||
|
||||
child.stderr?.on("data", (data) => {
|
||||
stderr += data;
|
||||
});
|
||||
|
||||
child.on("close", (code) => {
|
||||
if (code !== 0) {
|
||||
reject(new Error(`Error code: ${code}\nstderr: ${stderr}`));
|
||||
} else {
|
||||
resolve({ stdout, stderr });
|
||||
}
|
||||
});
|
||||
|
||||
child.on("error", (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
interface FCResponse {
|
||||
statusCode: number;
|
||||
body: string;
|
||||
}
|
||||
|
||||
interface Proxy {
|
||||
@ -91,6 +72,25 @@ function shuffleArray<T>(array: T[]): T[] {
|
||||
return newArray;
|
||||
}
|
||||
|
||||
const getEndpoint = (region: string) => `fcv3.cn-${region}.aliyuncs.com`;
|
||||
|
||||
const getAlicloudClient = (region: string) => {
|
||||
const credential = new Credential();
|
||||
const config = new OpenApi.Config({
|
||||
credential: credential
|
||||
});
|
||||
config.endpoint = getEndpoint(region);
|
||||
return new FC20230330(config);
|
||||
};
|
||||
|
||||
const streamToString = async (readableStream: Readable) => {
|
||||
let data = "";
|
||||
for await (const chunk of readableStream) {
|
||||
data += chunk.toString();
|
||||
}
|
||||
return data;
|
||||
};
|
||||
|
||||
class NetworkDelegate {
|
||||
private proxies: ProxiesMap = {};
|
||||
private providerLimiters: LimiterMap = {};
|
||||
@ -119,7 +119,9 @@ class NetworkDelegate {
|
||||
const proxies = this.getTaskProxies(taskName);
|
||||
for (const proxyName of proxies) {
|
||||
const limiterId = "proxy-" + proxyName + "-" + taskName;
|
||||
this.proxyLimiters[limiterId] = config ? new MultipleRateLimiter(limiterId, config) : null;
|
||||
this.proxyLimiters[limiterId] = config
|
||||
? new MultipleRateLimiter(limiterId, config)
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
@ -165,12 +167,12 @@ class NetworkDelegate {
|
||||
* - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
|
||||
* - The proxy type is not supported: with error code `NOT_IMPLEMENTED`
|
||||
*/
|
||||
async request<R>(url: string, task: string, method: string = "GET"): Promise<R> {
|
||||
async request<R>(url: string, task: string): Promise<R> {
|
||||
// find a available proxy
|
||||
const proxiesNames = this.getTaskProxies(task);
|
||||
for (const proxyName of shuffleArray(proxiesNames)) {
|
||||
try {
|
||||
return await this.proxyRequest<R>(url, proxyName, task, method);
|
||||
return await this.proxyRequest<R>(url, proxyName, task);
|
||||
} catch (e) {
|
||||
if (e instanceof RateLimiterError) {
|
||||
continue;
|
||||
@ -200,7 +202,6 @@ class NetworkDelegate {
|
||||
url: string,
|
||||
proxyName: string,
|
||||
task: string,
|
||||
method: string = "GET",
|
||||
force: boolean = false
|
||||
): Promise<R> {
|
||||
const proxy = this.proxies[proxyName];
|
||||
@ -209,28 +210,30 @@ class NetworkDelegate {
|
||||
}
|
||||
|
||||
await this.triggerLimiter(task, proxyName, force);
|
||||
const result = await this.makeRequest<R>(url, proxy, method);
|
||||
const result = await this.makeRequest<R>(url, proxy);
|
||||
return result;
|
||||
}
|
||||
|
||||
private async makeRequest<R>(url: string, proxy: Proxy, method: string): Promise<R> {
|
||||
private async makeRequest<R>(url: string, proxy: Proxy): Promise<R> {
|
||||
switch (proxy.type) {
|
||||
case "native":
|
||||
return await this.nativeRequest<R>(url, method);
|
||||
return await this.nativeRequest<R>(url);
|
||||
case "alicloud-fc":
|
||||
return await this.alicloudFcRequest<R>(url, proxy.data);
|
||||
default:
|
||||
throw new NetSchedulerError(`Proxy type ${proxy.type} not supported`, "NOT_IMPLEMENTED");
|
||||
throw new NetSchedulerError(
|
||||
`Proxy type ${proxy.type} not supported`,
|
||||
"NOT_IMPLEMENTED"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private async nativeRequest<R>(url: string, method: string): Promise<R> {
|
||||
private async nativeRequest<R>(url: string): Promise<R> {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), 10 * SECOND);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
signal: controller.signal
|
||||
});
|
||||
|
||||
@ -244,31 +247,31 @@ class NetworkDelegate {
|
||||
|
||||
private async alicloudFcRequest<R>(url: string, region: string): Promise<R> {
|
||||
try {
|
||||
const output = await spawnPromise("aliyun", [
|
||||
"fc",
|
||||
"POST",
|
||||
`/2023-03-30/functions/proxy-${region}/invocations`,
|
||||
"--qualifier",
|
||||
"LATEST",
|
||||
"--header",
|
||||
"Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;",
|
||||
"--body",
|
||||
JSON.stringify({ url: url }),
|
||||
"--retry-count",
|
||||
"5",
|
||||
"--read-timeout",
|
||||
"30",
|
||||
"--connect-timeout",
|
||||
"10",
|
||||
"--profile",
|
||||
`CVSA-${region}`
|
||||
]);
|
||||
const out = output.stdout;
|
||||
const rawData = JSON.parse(out);
|
||||
const client = getAlicloudClient(region);
|
||||
const bodyStream = Stream.readFromString(JSON.stringify({ url: url }));
|
||||
const headers = new $FC20230330.InvokeFunctionHeaders({});
|
||||
const request = new $FC20230330.InvokeFunctionRequest({
|
||||
body: bodyStream
|
||||
});
|
||||
const runtime = new Util.RuntimeOptions({});
|
||||
const response = await client.invokeFunctionWithOptions(
|
||||
`proxy-${region}`,
|
||||
request,
|
||||
headers,
|
||||
runtime
|
||||
);
|
||||
if (response.statusCode !== 200) {
|
||||
// noinspection ExceptionCaughtLocallyJS
|
||||
throw new NetSchedulerError(
|
||||
`Error proxying ${url} to ali-fc region ${region}, code: ${response.statusCode} (Not correctly invoked).`,
|
||||
"ALICLOUD_PROXY_ERR"
|
||||
);
|
||||
}
|
||||
const rawData = JSON.parse(await streamToString(response.body)) as FCResponse;
|
||||
if (rawData.statusCode !== 200) {
|
||||
// noinspection ExceptionCaughtLocallyJS
|
||||
throw new NetSchedulerError(
|
||||
`Error proxying ${url} to ali-fc region ${region}, code: ${rawData.statusCode}.`,
|
||||
`Error proxying ${url} to ali-fc region ${region}, code: ${rawData.statusCode}. (fetch error)`,
|
||||
"ALICLOUD_PROXY_ERR"
|
||||
);
|
||||
} else {
|
||||
@ -319,7 +322,7 @@ const biliLimiterConfig: RateLimiterConfig[] = [
|
||||
},
|
||||
{
|
||||
duration: 5 * 60,
|
||||
max: 200
|
||||
max: 500
|
||||
}
|
||||
];
|
||||
|
||||
@ -358,11 +361,21 @@ The order of setTaskLimiter and setProviderLimiter relative to each other is fle
|
||||
but both should come after addProxy and addTask to ensure proper setup and dependencies are met.
|
||||
*/
|
||||
|
||||
const regions = ["shanghai", "hangzhou", "qingdao", "beijing", "zhangjiakou", "chengdu", "shenzhen", "hohhot"];
|
||||
const regions = [
|
||||
"shanghai",
|
||||
"hangzhou",
|
||||
"qingdao",
|
||||
"beijing",
|
||||
"zhangjiakou",
|
||||
"chengdu",
|
||||
"shenzhen",
|
||||
"hohhot"
|
||||
];
|
||||
networkDelegate.addProxy("native", "native", "");
|
||||
for (const region of regions) {
|
||||
networkDelegate.addProxy(`alicloud-${region}`, "alicloud-fc", region);
|
||||
}
|
||||
networkDelegate.addTask("test", "test", "all");
|
||||
networkDelegate.addTask("getVideoInfo", "bilibili", "all");
|
||||
networkDelegate.addTask("getLatestVideos", "bilibili", "all");
|
||||
networkDelegate.addTask(
|
||||
@ -391,6 +404,8 @@ networkDelegate.setTaskLimiter("getLatestVideos", null);
|
||||
networkDelegate.setTaskLimiter("snapshotMilestoneVideo", null);
|
||||
networkDelegate.setTaskLimiter("snapshotVideo", null);
|
||||
networkDelegate.setTaskLimiter("bulkSnapshot", null);
|
||||
networkDelegate.setTaskLimiter("test", null);
|
||||
networkDelegate.setProviderLimiter("test", []);
|
||||
networkDelegate.setProviderLimiter("bilibili", biliLimiterConfig);
|
||||
networkDelegate.setProviderLimiter("bili_test", bili_test);
|
||||
networkDelegate.setProviderLimiter("bili_strict", bili_strict);
|
||||
|
||||
@ -7,6 +7,11 @@
|
||||
"build": "bun build ./index.ts --target node --outdir ./dist"
|
||||
},
|
||||
"dependencies": {
|
||||
"@alicloud/credentials": "^2.4.4",
|
||||
"@alicloud/darabonba-stream": "^0.0.2",
|
||||
"@alicloud/fc20230330": "^4.6.2",
|
||||
"@alicloud/openapi-client": "^0.4.15",
|
||||
"@alicloud/tea-util": "^1.4.10",
|
||||
"@koshnic/ratelimit": "^1.0.3",
|
||||
"@types/luxon": "^3.7.1",
|
||||
"chalk": "^5.4.1",
|
||||
|
||||
9
packages/core/test/netDelegate.test.ts
Normal file
9
packages/core/test/netDelegate.test.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import networkDelegate from "@core/net/delegate";
|
||||
import { test, expect, describe } from "bun:test";
|
||||
|
||||
describe("proxying requests", () => {
|
||||
test("Alibaba Cloud FC", async () => {
|
||||
const res = await networkDelegate.request("https://postman-echo.com/get", "test") as any;
|
||||
expect(res.headers.referer).toBe('https://www.bilibili.com/');
|
||||
});
|
||||
});
|
||||
@ -1,8 +1,8 @@
|
||||
import type { SnapshotScheduleType } from "@core/db/schema.d";
|
||||
import logger from "@core/log";
|
||||
import { MINUTE } from "@core/lib";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { Redis } from "ioredis";
|
||||
import { redis } from "bun";
|
||||
import { RedisClient } from "bun";
|
||||
import { parseTimestampFromPsql } from "../utils/formatTimestampToPostgre";
|
||||
import type { Psql } from "@core/db/psql.d";
|
||||
|
||||
@ -14,7 +14,7 @@ function getCurrentWindowIndex(): number {
|
||||
return Math.floor(minutesSinceMidnight / 5);
|
||||
}
|
||||
|
||||
export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: Redis) {
|
||||
export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: RedisClient) {
|
||||
const now = new Date();
|
||||
const startTime = now.getTime();
|
||||
|
||||
@ -37,19 +37,19 @@ export async function refreshSnapshotWindowCounts(sql: Psql, redisClient: Redis)
|
||||
const targetOffset = Math.floor((row.window_start.getTime() - startTime) / (5 * MINUTE));
|
||||
const offset = currentWindow + targetOffset;
|
||||
if (offset >= 0) {
|
||||
await redisClient.hset(REDIS_KEY, offset.toString(), Number(row.count));
|
||||
await redisClient.hmset(REDIS_KEY, [offset.toString(), row.count.toString()]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function initSnapshotWindowCounts(sql: Psql, redisClient: Redis) {
|
||||
export async function initSnapshotWindowCounts(sql: Psql, redisClient: RedisClient) {
|
||||
await refreshSnapshotWindowCounts(sql, redisClient);
|
||||
setInterval(async () => {
|
||||
await refreshSnapshotWindowCounts(sql, redisClient);
|
||||
}, 5 * MINUTE);
|
||||
}
|
||||
|
||||
async function getWindowCount(redisClient: Redis, offset: number): Promise<number> {
|
||||
async function getWindowCount(redisClient: RedisClient, offset: number): Promise<number> {
|
||||
const count = await redisClient.hget(REDIS_KEY, offset.toString());
|
||||
return count ? parseInt(count, 10) : 0;
|
||||
}
|
||||
@ -239,7 +239,7 @@ export async function bulkScheduleSnapshot(
|
||||
export async function adjustSnapshotTime(
|
||||
expectedStartTime: Date,
|
||||
allowedCounts: number = 1000,
|
||||
redisClient: Redis
|
||||
redisClient: RedisClient
|
||||
): Promise<Date> {
|
||||
const currentWindow = getCurrentWindowIndex();
|
||||
const targetOffset = Math.floor((expectedStartTime.getTime() - Date.now()) / (5 * MINUTE)) - 6;
|
||||
|
||||
@ -1,46 +1,13 @@
|
||||
import { Job } from "bullmq";
|
||||
import { sql } from "@core/db/dbNew";
|
||||
import logger from "@core/log";
|
||||
import { scheduleSnapshot, setSnapshotStatus } from "db/snapshotSchedule";
|
||||
import { SECOND } from "@core/lib";
|
||||
import { getTimeoutSchedulesCount } from "mq/task/getTimeoutSchedulesCount";
|
||||
import { removeAllTimeoutSchedules } from "mq/task/removeAllTimeoutSchedules";
|
||||
|
||||
interface SnapshotSchedule {
|
||||
id: bigint;
|
||||
aid: bigint;
|
||||
type: string;
|
||||
}
|
||||
|
||||
export const scheduleCleanupWorker = async (_job: Job): Promise<void> => {
|
||||
try {
|
||||
if ((await getTimeoutSchedulesCount()) > 2000) {
|
||||
await removeAllTimeoutSchedules();
|
||||
return;
|
||||
}
|
||||
|
||||
const rows = await sql<SnapshotSchedule[]>`
|
||||
SELECT id, aid, type
|
||||
FROM snapshot_schedule
|
||||
WHERE status IN ('pending', 'processing')
|
||||
AND started_at < NOW() - INTERVAL '30 minutes'
|
||||
UNION
|
||||
SELECT id, aid, type
|
||||
FROM snapshot_schedule
|
||||
WHERE status IN ('pending', 'processing')
|
||||
AND started_at < NOW() - INTERVAL '2 minutes'
|
||||
AND type = 'milestone'
|
||||
`;
|
||||
|
||||
if (rows.length === 0) return;
|
||||
for (const row of rows) {
|
||||
const id = Number(row.id);
|
||||
const aid = Number(row.aid);
|
||||
const type = row.type;
|
||||
await setSnapshotStatus(sql, id, "timeout");
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + 10 * SECOND);
|
||||
const row = await removeAllTimeoutSchedules();
|
||||
if (row.length > 0 && row[0].deleted) {
|
||||
logger.log(
|
||||
`Schedule ${id} has not received any response in a while, rescheduled.`,
|
||||
`Removed ${row[0].deleted} timeout schedules.`,
|
||||
"mq",
|
||||
"fn:scheduleCleanupWorker"
|
||||
);
|
||||
|
||||
@ -1,5 +1,10 @@
|
||||
import { Job } from "bullmq";
|
||||
import { getLatestSnapshot, scheduleSnapshot, setSnapshotStatus, snapshotScheduleExists } from "db/snapshotSchedule";
|
||||
import {
|
||||
getLatestSnapshot,
|
||||
scheduleSnapshot,
|
||||
setSnapshotStatus,
|
||||
snapshotScheduleExists
|
||||
} from "db/snapshotSchedule";
|
||||
import logger from "@core/log";
|
||||
import { HOUR, MINUTE, SECOND } from "@core/lib";
|
||||
import { getBiliVideoStatus, setBiliVideoStatus } from "../../db/bilibili_metadata";
|
||||
@ -28,6 +33,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
if (!exists) {
|
||||
return;
|
||||
}
|
||||
|
||||
const status = await getBiliVideoStatus(sql, aid);
|
||||
if (status !== 0) {
|
||||
logger.warn(
|
||||
@ -37,6 +43,7 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await setSnapshotStatus(sql, id, "processing");
|
||||
const stat = await insertVideoSnapshot(sql, aid, task);
|
||||
if (typeof stat === "number") {
|
||||
@ -94,7 +101,11 @@ export const snapshotVideoWorker = async (job: Job): Promise<void> => {
|
||||
return;
|
||||
} catch (e) {
|
||||
if (e instanceof NetSchedulerError && e.code === "NO_PROXY_AVAILABLE") {
|
||||
logger.warn(`No available proxy for aid ${job.data.aid}.`, "mq", "fn:snapshotVideoWorker");
|
||||
logger.warn(
|
||||
`No available proxy for aid ${job.data.aid}.`,
|
||||
"mq",
|
||||
"fn:snapshotVideoWorker"
|
||||
);
|
||||
await setSnapshotStatus(sql, id, "no_proxy");
|
||||
await scheduleSnapshot(sql, aid, type, Date.now() + retryInterval, false, true);
|
||||
return;
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Queue, ConnectionOptions } from "bullmq";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis } from "bun";
|
||||
|
||||
export const LatestVideosQueue = new Queue("latestVideos", {
|
||||
connection: redis as ConnectionOptions
|
||||
|
||||
@ -2,7 +2,7 @@ import { HOUR, MINUTE, SECOND } from "@core/lib";
|
||||
import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index";
|
||||
import logger from "@core/log";
|
||||
import { initSnapshotWindowCounts } from "db/snapshotSchedule";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis } from "bun";
|
||||
import { sql } from "@core/db/dbNew";
|
||||
|
||||
export async function initMQ() {
|
||||
|
||||
@ -2,10 +2,18 @@ import { sql } from "@core/db/dbNew";
|
||||
import logger from "@core/log";
|
||||
|
||||
export async function removeAllTimeoutSchedules() {
|
||||
logger.log("Too many timeout schedules, directly removing these schedules...", "mq", "fn:scheduleCleanupWorker");
|
||||
logger.log(
|
||||
"Too many timeout schedules, directly removing these schedules...",
|
||||
"mq",
|
||||
"fn:removeAllTimeoutSchedules"
|
||||
);
|
||||
return sql`
|
||||
DELETE FROM snapshot_schedule
|
||||
WHERE status IN ('pending', 'processing')
|
||||
AND started_at < NOW() - INTERVAL '30 minutes'
|
||||
WITH deleted AS (
|
||||
DELETE FROM snapshot_schedule
|
||||
WHERE status IN ('pending', 'processing')
|
||||
AND started_at < NOW() - INTERVAL '30 minutes'
|
||||
RETURNING *
|
||||
)
|
||||
SELECT count(*) FROM deleted;
|
||||
`;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { ConnectionOptions, Job, Worker } from "bullmq";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis } from "bun";
|
||||
import logger from "@core/log";
|
||||
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo";
|
||||
import { WorkerError } from "mq/schema";
|
||||
|
||||
@ -12,7 +12,7 @@ import {
|
||||
snapshotVideoWorker,
|
||||
takeBulkSnapshotForVideosWorker
|
||||
} from "mq/exec/executors";
|
||||
import { redis } from "@core/db/redis";
|
||||
import { redis } from "bun";
|
||||
import logger from "@core/log";
|
||||
import { lockManager } from "@core/mq/lockManager";
|
||||
import { WorkerError } from "mq/schema";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user