alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: bilibili metadata fetching

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-01-11 20:29:19 +08:00
parent 5d2f61bd73
commit 75110be3c1
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
7 changed files with 650409 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -63,4 +63,7 @@ TEST-results.xml
package-lock.json
.eslintcache
*v8.log
/lib/
/lib/
# project specific
data/main.db

649902
data/2025010104_c30_aids.txt Normal file
View File

File diff suppressed because it is too large Load Diff

3
data/README.md Normal file
View File

@ -0,0 +1,3 @@
# The data
感谢[天钿Daily](https://tdd.bunnyxt.com/)提供的数据。

11
deno.json Normal file
View File

@ -0,0 +1,11 @@
{
"tasks": {
"crawl-raw-bili": "deno --allow-env --allow-ffi --allow-read --allow-net src/db/raw/insertAidsToDB.ts"
},
"imports": {
"@std/assert": "jsr:@std/assert@1",
"@types/better-sqlite3": "npm:@types/better-sqlite3@^7.6.12",
"axios": "npm:axios@^1.7.9",
"better-sqlite3": "npm:better-sqlite3@^11.7.2"
}
}

342
deno.lock Normal file
View File

@ -0,0 +1,342 @@
{
"version": "4",
"specifiers": {
"jsr:@db/sqlite@0.12": "0.12.0",
"jsr:@denosaurs/plug@1": "1.0.6",
"jsr:@std/assert@0.217": "0.217.0",
"jsr:@std/assert@0.221": "0.221.0",
"jsr:@std/assert@1": "1.0.10",
"jsr:@std/encoding@0.221": "0.221.0",
"jsr:@std/fmt@0.221": "0.221.0",
"jsr:@std/fs@0.221": "0.221.0",
"jsr:@std/internal@^1.0.5": "1.0.5",
"jsr:@std/path@0.217": "0.217.0",
"jsr:@std/path@0.221": "0.221.0",
"npm:@types/better-sqlite3@*": "7.6.12",
"npm:@types/better-sqlite3@^7.6.12": "7.6.12",
"npm:@types/node@*": "22.5.4",
"npm:axios@^1.7.9": "1.7.9",
"npm:better-sqlite3@^11.7.2": "11.7.2"
},
"jsr": {
"@db/sqlite@0.12.0": {
"integrity": "dd1ef7f621ad50fc1e073a1c3609c4470bd51edc0994139c5bf9851de7a6d85f",
"dependencies": [
"jsr:@denosaurs/plug",
"jsr:@std/path@0.217"
]
},
"@denosaurs/plug@1.0.6": {
"integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7",
"dependencies": [
"jsr:@std/encoding",
"jsr:@std/fmt",
"jsr:@std/fs",
"jsr:@std/path@0.221"
]
},
"@std/assert@0.217.0": {
"integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642"
},
"@std/assert@0.221.0": {
"integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a"
},
"@std/assert@1.0.10": {
"integrity": "59b5cbac5bd55459a19045d95cc7c2ff787b4f8527c0dd195078ff6f9481fbb3",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/encoding@0.221.0": {
"integrity": "d1dd76ef0dc5d14088411e6dc1dede53bf8308c95d1537df1214c97137208e45"
},
"@std/fmt@0.221.0": {
"integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a"
},
"@std/fs@0.221.0": {
"integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286",
"dependencies": [
"jsr:@std/assert@0.221",
"jsr:@std/path@0.221"
]
},
"@std/internal@1.0.5": {
"integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba"
},
"@std/path@0.217.0": {
"integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11",
"dependencies": [
"jsr:@std/assert@0.217"
]
},
"@std/path@0.221.0": {
"integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095",
"dependencies": [
"jsr:@std/assert@0.221"
]
}
},
"npm": {
"@types/better-sqlite3@7.6.12": {
"integrity": "sha512-fnQmj8lELIj7BSrZQAdBMHEHX8OZLYIHXqAKT1O7tDfLxaINzf00PMjw22r3N/xXh0w/sGHlO6SVaCQ2mj78lg==",
"dependencies": [
"@types/node"
]
},
"@types/node@22.5.4": {
"integrity": "sha512-FDuKUJQm/ju9fT/SeX/6+gBzoPzlVCzfzmGkwKvRHQVxi4BntVbyIwf6a4Xn62mrvndLiml6z/UBXIdEVjQLXg==",
"dependencies": [
"undici-types"
]
},
"asynckit@0.4.0": {
"integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q=="
},
"axios@1.7.9": {
"integrity": "sha512-LhLcE7Hbiryz8oMDdDptSrWowmB4Bl6RCt6sIJKpRB4XtVf0iEgewX3au/pJqm+Py1kCASkb/FFKjxQaLtxJvw==",
"dependencies": [
"follow-redirects",
"form-data",
"proxy-from-env"
]
},
"base64-js@1.5.1": {
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA=="
},
"better-sqlite3@11.7.2": {
"integrity": "sha512-10a57cHVDmfNQS4jrZ9AH2t+2ekzYh5Rhbcnb4ytpmYweoLdogDmyTt5D+hLiY9b44Mx9foowb/4iXBTO2yP3Q==",
"dependencies": [
"bindings",
"prebuild-install"
]
},
"bindings@1.5.0": {
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"dependencies": [
"file-uri-to-path"
]
},
"bl@4.1.0": {
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"dependencies": [
"buffer",
"inherits",
"readable-stream"
]
},
"buffer@5.7.1": {
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"dependencies": [
"base64-js",
"ieee754"
]
},
"chownr@1.1.4": {
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"combined-stream@1.0.8": {
"integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
"dependencies": [
"delayed-stream"
]
},
"decompress-response@6.0.0": {
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"dependencies": [
"mimic-response"
]
},
"deep-extend@0.6.0": {
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA=="
},
"delayed-stream@1.0.0": {
"integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ=="
},
"detect-libc@2.0.3": {
"integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw=="
},
"end-of-stream@1.4.4": {
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"dependencies": [
"once"
]
},
"expand-template@2.0.3": {
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg=="
},
"file-uri-to-path@1.0.0": {
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw=="
},
"follow-redirects@1.15.9": {
"integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ=="
},
"form-data@4.0.1": {
"integrity": "sha512-tzN8e4TX8+kkxGPK8D5u0FNmjPUjw3lwC9lSLxxoB/+GtsJG91CO8bSWy73APlgAZzZbXEYZJuxjkHH2w+Ezhw==",
"dependencies": [
"asynckit",
"combined-stream",
"mime-types"
]
},
"fs-constants@1.0.0": {
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"github-from-package@0.0.0": {
"integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw=="
},
"ieee754@1.2.1": {
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="
},
"inherits@2.0.4": {
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
},
"ini@1.3.8": {
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew=="
},
"mime-db@1.52.0": {
"integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg=="
},
"mime-types@2.1.35": {
"integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
"dependencies": [
"mime-db"
]
},
"mimic-response@3.1.0": {
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ=="
},
"minimist@1.2.8": {
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="
},
"mkdirp-classic@0.5.3": {
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
},
"napi-build-utils@1.0.2": {
"integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg=="
},
"node-abi@3.71.0": {
"integrity": "sha512-SZ40vRiy/+wRTf21hxkkEjPJZpARzUMVcJoQse2EF8qkUWbbO2z7vd5oA/H6bVH6SZQ5STGcu0KRDS7biNRfxw==",
"dependencies": [
"semver"
]
},
"once@1.4.0": {
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"dependencies": [
"wrappy"
]
},
"prebuild-install@7.1.2": {
"integrity": "sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ==",
"dependencies": [
"detect-libc",
"expand-template",
"github-from-package",
"minimist",
"mkdirp-classic",
"napi-build-utils",
"node-abi",
"pump",
"rc",
"simple-get",
"tar-fs",
"tunnel-agent"
]
},
"proxy-from-env@1.1.0": {
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"pump@3.0.2": {
"integrity": "sha512-tUPXtzlGM8FE3P0ZL6DVs/3P58k9nk8/jZeQCurTJylQA8qFYzHFfhBJkuqyE0FifOsQ0uKWekiZ5g8wtr28cw==",
"dependencies": [
"end-of-stream",
"once"
]
},
"rc@1.2.8": {
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"dependencies": [
"deep-extend",
"ini",
"minimist",
"strip-json-comments"
]
},
"readable-stream@3.6.2": {
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"dependencies": [
"inherits",
"string_decoder",
"util-deprecate"
]
},
"safe-buffer@5.2.1": {
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"semver@7.6.3": {
"integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A=="
},
"simple-concat@1.0.1": {
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q=="
},
"simple-get@4.0.1": {
"integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
"dependencies": [
"decompress-response",
"once",
"simple-concat"
]
},
"string_decoder@1.3.0": {
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"dependencies": [
"safe-buffer"
]
},
"strip-json-comments@2.0.1": {
"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="
},
"tar-fs@2.1.1": {
"integrity": "sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng==",
"dependencies": [
"chownr",
"mkdirp-classic",
"pump",
"tar-stream"
]
},
"tar-stream@2.2.0": {
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"dependencies": [
"bl",
"end-of-stream",
"fs-constants",
"inherits",
"readable-stream"
]
},
"tunnel-agent@0.6.0": {
"integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
"dependencies": [
"safe-buffer"
]
},
"undici-types@6.19.8": {
"integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw=="
},
"util-deprecate@1.0.2": {
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
},
"wrappy@1.0.2": {
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
}
},
"workspace": {
"dependencies": [
"jsr:@std/assert@1",
"npm:@types/better-sqlite3@^7.6.12",
"npm:axios@^1.7.9",
"npm:better-sqlite3@^11.7.2"
]
}
}

42
src/crawler/bilibili/videoInfo.ts Normal file
View File

@ -0,0 +1,42 @@
import axios from "axios";
// 随机选择 User-Agent
function getRandomUserAgent() {
const userAgents = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0.3 Safari/605.1.15",
"Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36",
"Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1",
// 可以添加更多的 User-Agent 字符串
];
const randomIndex = Math.floor(Math.random() * userAgents.length);
return userAgents[randomIndex];
}
export function getBiliBiliVideoInfo(bvidORaid?: string | number) {
const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined;
const aid = typeof bvidORaid === "number" ? bvidORaid : undefined;
if (!bvid && !aid) {
return null;
}
const baseURL = "https://api.bilibili.com/x/web-interface/view/detail";
const headers = {
'User-Agent': getRandomUserAgent(), // 添加随机 User-Agent
};
if (aid) {
return axios.get(baseURL, {
params: {
aid: aid,
},
headers: headers, // 将 headers 添加到请求中
});
} else {
return axios.get(baseURL, {
params: {
bvid: bvid,
},
headers: headers, // 将 headers 添加到请求中
});
}
}

105
src/db/raw/insertAidsToDB.ts Normal file
View File

@ -0,0 +1,105 @@
import path from "node:path";
import { Database } from "jsr:@db/sqlite@0.12";
import { getBiliBiliVideoInfo } from "../../crawler/bilibili/videoInfo.ts";
const aidPath = path.join("./data/2025010104_c30_aids.txt");
const db = new Database("./data/main.db");
async function insertAidsToDB() {
const aidRawcontent = await Deno.readTextFile(aidPath);
const aids = aidRawcontent
.split("\n")
.filter((line) => line.length > 0)
.map((line) => parseInt(line));
// Insert aids into database with status 'pending'
// if bili_info_crawl contains at least one row, skip it
if (db.prepare("SELECT * FROM bili_info_crawl").all().length == 0) {
const insertStmt = db.prepare(`
INSERT OR IGNORE INTO bili_info_crawl (aid, status)
VALUES (?, 'pending')
`);
for (const aid of aids) {
insertStmt.run(aid);
}
}
const aidsInDB = db
.prepare("SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'")
.all()
.map((row) => row.aid) as number[];
const totalAids = aidsInDB.length;
let processedAids = 0;
const startTime = Date.now();
// Update database with video info
for (const aid of aidsInDB) {
try {
const res = await getBiliBiliVideoInfo(aid);
if (res?.data.code !== 0) {
const data = res?.data;
db.prepare(
`
UPDATE bili_info_crawl
SET status = 'error',
data = ?
WHERE aid = ?
`
).run(aid, JSON.stringify(data));
} else {
const data = res.data.data;
db.prepare(
`
UPDATE bili_info_crawl
SET status = 'success',
bvid = ?,
data = ?
WHERE aid = ?
`
).run(data.View.bvid, JSON.stringify(data), aid);
}
} catch (error) {
console.error(`Error updating aid ${aid}: ${error}`);
try {
db.prepare(
`
UPDATE bili_info_crawl
SET status = 'failed'
WHERE aid = ?
`
).run(aid);
}
catch (error) {
console.error(`Error wrting to db for aid ${aid}: ${error}`);
}
} finally {
processedAids++;
const elapsedTime = Date.now() - startTime;
const elapsedSeconds = Math.floor(elapsedTime / 1000);
const elapsedMinutes = Math.floor(elapsedSeconds / 60);
const elapsedHours = Math.floor(elapsedMinutes / 60);
const remainingAids = totalAids - processedAids;
// Calculate ETA
const averageTimePerAid = elapsedTime / processedAids;
const eta = remainingAids * averageTimePerAid;
const etaSeconds = Math.floor(eta / 1000);
const etaMinutes = Math.floor(etaSeconds / 60);
const etaHours = Math.floor(etaMinutes / 60);
// Output progress
const progress = `${processedAids}/${totalAids}, ${(processedAids / totalAids * 100).toFixed(2)}%, elapsed ${elapsedHours.toString().padStart(2, '0')}:${(elapsedMinutes % 60).toString().padStart(2, '0')}:${(elapsedSeconds % 60).toString().padStart(2, '0')}, ETA ${etaHours}h${(etaMinutes % 60).toString().padStart(2, '0')}m`;
if (Math.random() > 0.95) {
console.log("Sleeping...");
const time = Math.random() * 5 * 1000;
await new Promise((resolve) => setTimeout(resolve, time));
}
console.log(`Updated aid ${aid}, ${progress}`);
}
}
}
insertAidsToDB();