diff --git a/.gitignore b/.gitignore index 883b2d5..d17200a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,3 @@ -node_modules/ -.node_modules/ built/* tests/cases/rwc/* tests/cases/perf/* @@ -75,4 +73,16 @@ data/filter/eval* data/filter/train* filter/checkpoints data/filter/model_predicted* -scripts/*.ipynb \ No newline at end of file +scripts + +# dotenv environment variable files +.env +.env.development.local +.env.test.local +.env.production.local +.env.local + +# Fresh build directory +_fresh/ +# npm dependencies +node_modules/ diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 0000000..ec71323 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,6 @@ +{ + "recommendations": [ + "denoland.vscode-deno", + "bradlc.vscode-tailwindcss" + ] +} diff --git a/components/Button.tsx b/components/Button.tsx new file mode 100644 index 0000000..6e868c5 --- /dev/null +++ b/components/Button.tsx @@ -0,0 +1,12 @@ +import { JSX } from "preact"; +import { IS_BROWSER } from "$fresh/runtime.ts"; + +export function Button(props: JSX.HTMLAttributes) { + return ( + +

{props.count}

+ + + ); +} diff --git a/lab/align-pipeline.md b/lab/align-pipeline.md index b140b9d..887b919 100644 --- a/lab/align-pipeline.md +++ b/lab/align-pipeline.md @@ -1,3 +1,3 @@ 1. prepare `1.mp3`, `1.txt`, `1.group` in `./data` 2. `whisperAlignment/alignWithGroup` -3. `mmsAlignment \ No newline at end of file +3. `mmsAlignment diff --git a/main.ts b/main.ts new file mode 100644 index 0000000..675f529 --- /dev/null +++ b/main.ts @@ -0,0 +1,13 @@ +/// +/// +/// +/// +/// + +import "$std/dotenv/load.ts"; + +import { start } from "$fresh/server.ts"; +import manifest from "./fresh.gen.ts"; +import config from "./fresh.config.ts"; + +await start(manifest, config); diff --git a/routes/_404.tsx b/routes/_404.tsx new file mode 100644 index 0000000..4628eeb --- /dev/null +++ b/routes/_404.tsx @@ -0,0 +1,27 @@ +import { Head } from "$fresh/runtime.ts"; + +export default function Error404() { + return ( + <> + + 404 - Page not found + +
+
+ the Fresh logo: a sliced lemon dripping with juice +

404 - Page not found

+

+ The page you were looking for doesn't exist. +

+ Go back home +
+
+ + ); +} diff --git a/routes/_app.tsx b/routes/_app.tsx new file mode 100644 index 0000000..a44414e --- /dev/null +++ b/routes/_app.tsx @@ -0,0 +1,16 @@ +import { type PageProps } from "$fresh/server.ts"; +export default function App({ Component }: PageProps) { + return ( + + + + + cvsa + + + + + + + ); +} diff --git a/routes/api/joke.ts b/routes/api/joke.ts new file mode 100644 index 0000000..68b0ebe --- /dev/null +++ b/routes/api/joke.ts @@ -0,0 +1,21 @@ +import { FreshContext } from "$fresh/server.ts"; + +// Jokes courtesy of https://punsandoneliners.com/randomness/programmer-jokes/ +const JOKES = [ + "Why do Java developers often wear glasses? They can't C#.", + "A SQL query walks into a bar, goes up to two tables and says “can I join you?”", + "Wasn't hard to crack Forrest Gump's password. 1forrest1.", + "I love pressing the F5 key. It's refreshing.", + "Called IT support and a chap from Australia came to fix my network connection. I asked “Do you come from a LAN down under?”", + "There are 10 types of people in the world. Those who understand binary and those who don't.", + "Why are assembly programmers often wet? They work below C level.", + "My favourite computer based band is the Black IPs.", + "What programme do you use to predict the music tastes of former US presidential candidates? An Al Gore Rhythm.", + "An SEO expert walked into a bar, pub, inn, tavern, hostelry, public house.", +]; + +export const handler = (_req: Request, _ctx: FreshContext): Response => { + const randomIndex = Math.floor(Math.random() * JOKES.length); + const body = JOKES[randomIndex]; + return new Response(body); +}; diff --git a/routes/greet/[name].tsx b/routes/greet/[name].tsx new file mode 100644 index 0000000..a7a5fe1 --- /dev/null +++ b/routes/greet/[name].tsx @@ -0,0 +1,5 @@ +import { PageProps } from "$fresh/server.ts"; + +export default function Greet(props: PageProps) { + return
Hello {props.params.name}
; +} diff --git a/routes/index.tsx b/routes/index.tsx new file mode 100644 index 0000000..67a22a7 --- /dev/null +++ b/routes/index.tsx @@ -0,0 +1,25 @@ +import { useSignal } from "@preact/signals"; +import Counter from "../islands/Counter.tsx"; + +export default function Home() { + const count = useSignal(3); + return ( +
+
+ the Fresh logo: a sliced lemon dripping with juice +

Welcome to Fresh

+

+ Try updating this message in the + ./routes/index.tsx file, and refresh. +

+ +
+
+ ); +} diff --git a/src/db/raw/aliyun-fc.mjs b/src/db/raw/aliyun-fc.mjs index 6b54405..0fcca42 100644 --- a/src/db/raw/aliyun-fc.mjs +++ b/src/db/raw/aliyun-fc.mjs @@ -1,77 +1,77 @@ -'use strict'; +"use strict"; export const handler = async (event, context) => { - const eventObj = JSON.parse(event); - console.log(`receive event: ${JSON.stringify(eventObj)}`); + const eventObj = JSON.parse(event); + console.log(`receive event: ${JSON.stringify(eventObj)}`); - let body = 'Missing parameter: URL'; - let statusCode = 400; + let body = "Missing parameter: URL"; + let statusCode = 400; - // User-Agent list - const userAgents = [ - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15', - 'Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36', - 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/89.0' - ]; + // User-Agent list + const userAgents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15", + "Mozilla/5.0 (Linux; Android 10; Pixel 3 XL) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Mobile Safari/537.36", + "Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.0 Mobile/15E148 Safari/604.1", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Gecko/20100101 Firefox/89.0", + ]; - // get http request body - if ("body" in eventObj) { - body = eventObj.body; - if (eventObj.isBase64Encoded) { - body = Buffer.from(body, 'base64').toString('utf-8'); - } - } - console.log(`receive http body: ${body}`); + // get http request body + if ("body" in eventObj) { + body = eventObj.body; + if (eventObj.isBase64Encoded) { + body = Buffer.from(body, "base64").toString("utf-8"); + } + } + console.log(`receive http body: ${body}`); - // proxy the URL if it exists in eventObj - const refererUrl = 'https://www.bilibili.com/'; // Replace with your desired referer and origin + // proxy the URL if it exists in eventObj + const refererUrl = "https://www.bilibili.com/"; // Replace with your desired referer and origin - if ("url" in eventObj) { - try { - const randomUserAgent = userAgents[Math.floor(Math.random() * userAgents.length)]; - const response = await fetch(eventObj.url, { - headers: { - 'User-Agent': randomUserAgent, - 'Referer': refererUrl - } - }); - statusCode = response.status; - body = await response.text(); - } catch (error) { - statusCode = 500; - body = `Error fetching URL: ${error.message}`; - } - } else if ("urls" in eventObj && Array.isArray(eventObj.urls)) { - const requests = eventObj.urls.map(async url => { - try { - const randomUserAgent = userAgents[Math.floor(Math.random() * userAgents.length)]; - const response = await fetch(url, { - headers: { - 'User-Agent': randomUserAgent, - 'Referer': refererUrl - } - }); - const responseBody = await response.text(); - return { - statusCode: response.status, - body: responseBody - }; - } catch (error) { - return { - statusCode: 500, - body: `Error fetching URL: ${error.message}` - }; - } - }); + if ("url" in eventObj) { + try { + const randomUserAgent = userAgents[Math.floor(Math.random() * userAgents.length)]; + const response = await fetch(eventObj.url, { + headers: { + "User-Agent": randomUserAgent, + "Referer": refererUrl, + }, + }); + statusCode = response.status; + body = await response.text(); + } catch (error) { + statusCode = 500; + body = `Error fetching URL: ${error.message}`; + } + } else if ("urls" in eventObj && Array.isArray(eventObj.urls)) { + const requests = eventObj.urls.map(async (url) => { + try { + const randomUserAgent = userAgents[Math.floor(Math.random() * userAgents.length)]; + const response = await fetch(url, { + headers: { + "User-Agent": randomUserAgent, + "Referer": refererUrl, + }, + }); + const responseBody = await response.text(); + return { + statusCode: response.status, + body: responseBody, + }; + } catch (error) { + return { + statusCode: 500, + body: `Error fetching URL: ${error.message}`, + }; + } + }); - body = await Promise.all(requests); - statusCode = 200; // Assuming all URLs were processed successfully - } + body = await Promise.all(requests); + statusCode = 200; // Assuming all URLs were processed successfully + } - return { - 'statusCode': statusCode, - 'body': JSON.stringify(body) - }; + return { + "statusCode": statusCode, + "body": JSON.stringify(body), + }; }; diff --git a/src/db/raw/fetchAids.ts b/src/db/raw/fetchAids.ts index 437f89a..10770d5 100644 --- a/src/db/raw/fetchAids.ts +++ b/src/db/raw/fetchAids.ts @@ -13,98 +13,98 @@ const db = new Database(DATABASE_PATH, { int64: true }); // 设置日志 async function setupLogging() { - await ensureDir(LOG_DIR); - const logStream = await Deno.open(LOG_FILE, { write: true, create: true, append: true }); + await ensureDir(LOG_DIR); + const logStream = await Deno.open(LOG_FILE, { write: true, create: true, append: true }); - const redirectConsole = - // deno-lint-ignore no-explicit-any - (originalConsole: (...args: any[]) => void) => - // deno-lint-ignore no-explicit-any - (...args: any[]) => { - const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" "); - originalConsole(message); - logStream.write(new TextEncoder().encode(message + "\n")); - }; + const redirectConsole = + // deno-lint-ignore no-explicit-any + (originalConsole: (...args: any[]) => void) => + // deno-lint-ignore no-explicit-any + (...args: any[]) => { + const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" "); + originalConsole(message); + logStream.write(new TextEncoder().encode(message + "\n")); + }; - console.log = redirectConsole(console.log); - console.error = redirectConsole(console.error); - console.warn = redirectConsole(console.warn); + console.log = redirectConsole(console.log); + console.error = redirectConsole(console.error); + console.warn = redirectConsole(console.warn); } interface Metadata { - key: string; - value: string; + key: string; + value: string; } // 获取最后一次更新的时间 function getLastUpdate(): Date { - const result = db.prepare("SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'").get() as Metadata; - return result ? new Date(result.value as string) : new Date(0); + const result = db.prepare("SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'").get() as Metadata; + return result ? new Date(result.value as string) : new Date(0); } // 更新最后更新时间 function updateLastUpdate() { - const now = new Date().toISOString(); - db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'").run(now); + const now = new Date().toISOString(); + db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'").run(now); } // 辅助函数：获取数据 // deno-lint-ignore no-explicit-any async function fetchData(pn: number, retries = MAX_RETRIES): Promise { - try { - const response = await fetch(`${API_URL}${pn}`); - if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); - return await response.json(); - } catch (error) { - if (retries > 0) { - await new Promise((resolve) => setTimeout(resolve, 1000)); - return fetchData(pn, retries - 1); - } - throw error; - } + try { + const response = await fetch(`${API_URL}${pn}`); + if (!response.ok) throw new Error(`HTTP error! status: ${response.status}`); + return await response.json(); + } catch (error) { + if (retries > 0) { + await new Promise((resolve) => setTimeout(resolve, 1000)); + return fetchData(pn, retries - 1); + } + throw error; + } } // 插入 aid 到数据库 function insertAid(aid: number) { - db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')").run(aid); + db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')").run(aid); } // 主函数 async function main() { - await setupLogging(); + await setupLogging(); - let pn = 1; - let shouldContinue = true; - const lastUpdate = getLastUpdate(); + let pn = 1; + let shouldContinue = true; + const lastUpdate = getLastUpdate(); - while (shouldContinue) { - try { - const data = await fetchData(pn); - const archives = data.data.archives; + while (shouldContinue) { + try { + const data = await fetchData(pn); + const archives = data.data.archives; - for (const archive of archives) { - const pubTime = new Date(archive.pubdate * 1000); - if (pubTime > lastUpdate) { - insertAid(archive.aid); - } else { - shouldContinue = false; - break; - } - } + for (const archive of archives) { + const pubTime = new Date(archive.pubdate * 1000); + if (pubTime > lastUpdate) { + insertAid(archive.aid); + } else { + shouldContinue = false; + break; + } + } - pn++; - console.log(`Fetched page ${pn}`); - } catch (error) { - console.error(`Error fetching data for pn=${pn}: ${error}`); - } - } + pn++; + console.log(`Fetched page ${pn}`); + } catch (error) { + console.error(`Error fetching data for pn=${pn}: ${error}`); + } + } - // 更新最后更新时间 - updateLastUpdate(); + // 更新最后更新时间 + updateLastUpdate(); - // 关闭数据库 - db.close(); + // 关闭数据库 + db.close(); } // 运行主函数 -main().catch(console.error); \ No newline at end of file +main().catch(console.error); diff --git a/src/db/raw/insertAidsToDB.ts b/src/db/raw/insertAidsToDB.ts index c9cad5a..b067d69 100644 --- a/src/db/raw/insertAidsToDB.ts +++ b/src/db/raw/insertAidsToDB.ts @@ -17,127 +17,127 @@ const MINUTES = MINUTE; const IPs = regions.length; const rateLimits = [ - { window: 5 * MINUTES, maxRequests: 160 * IPs }, - { window: 30 * SECONDS, maxRequests: 20 * IPs }, - { window: 1.2 * SECOND, maxRequests: 1 * IPs }, + { window: 5 * MINUTES, maxRequests: 160 * IPs }, + { window: 30 * SECONDS, maxRequests: 20 * IPs }, + { window: 1.2 * SECOND, maxRequests: 1 * IPs }, ]; const requestQueue: number[] = []; async function setupLogging() { - await ensureDir(logDir); - const logStream = await Deno.open(logFile, { write: true, create: true, append: true }); + await ensureDir(logDir); + const logStream = await Deno.open(logFile, { write: true, create: true, append: true }); - const redirectConsole = - // deno-lint-ignore no-explicit-any - (originalConsole: (...args: any[]) => void) => - // deno-lint-ignore no-explicit-any - (...args: any[]) => { - const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" "); - originalConsole(message); - logStream.write(new TextEncoder().encode(message + "\n")); - }; + const redirectConsole = + // deno-lint-ignore no-explicit-any + (originalConsole: (...args: any[]) => void) => + // deno-lint-ignore no-explicit-any + (...args: any[]) => { + const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" "); + originalConsole(message); + logStream.write(new TextEncoder().encode(message + "\n")); + }; - console.log = redirectConsole(console.log); - console.error = redirectConsole(console.error); - console.warn = redirectConsole(console.warn); + console.log = redirectConsole(console.log); + console.error = redirectConsole(console.error); + console.warn = redirectConsole(console.warn); } function isRateLimited(): boolean { - const now = Date.now(); - return rateLimits.some(({ window, maxRequests }) => { - const windowStart = now - window; - const requestsInWindow = requestQueue.filter((timestamp) => timestamp >= windowStart).length; - return requestsInWindow >= maxRequests; - }); + const now = Date.now(); + return rateLimits.some(({ window, maxRequests }) => { + const windowStart = now - window; + const requestsInWindow = requestQueue.filter((timestamp) => timestamp >= windowStart).length; + return requestsInWindow >= maxRequests; + }); } async function readFromText() { - const aidRawcontent = await Deno.readTextFile(aidPath); - const aids = aidRawcontent - .split("\n") - .filter((line) => line.length > 0) - .map((line) => parseInt(line)); + const aidRawcontent = await Deno.readTextFile(aidPath); + const aids = aidRawcontent + .split("\n") + .filter((line) => line.length > 0) + .map((line) => parseInt(line)); - // if (!db.prepare("SELECT COUNT(*) FROM bili_info_crawl").get()) { - // const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')"); - // aids.forEach((aid) => insertStmt.run(aid)); - // } + // if (!db.prepare("SELECT COUNT(*) FROM bili_info_crawl").get()) { + // const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')"); + // aids.forEach((aid) => insertStmt.run(aid)); + // } - // 查询数据库中已经存在的 aid - const existingAids = db - .prepare("SELECT aid FROM bili_info_crawl") - .all() - .map((row) => row.aid); - console.log(existingAids.length); + // 查询数据库中已经存在的 aid + const existingAids = db + .prepare("SELECT aid FROM bili_info_crawl") + .all() + .map((row) => row.aid); + console.log(existingAids.length); - // 将 existingAids 转换为 Set 以提高查找效率 - const existingAidsSet = new Set(existingAids); + // 将 existingAids 转换为 Set 以提高查找效率 + const existingAidsSet = new Set(existingAids); - // 找出 aids 数组中不存在于数据库的条目 - const newAids = aids.filter((aid) => !existingAidsSet.has(aid)); + // 找出 aids 数组中不存在于数据库的条目 + const newAids = aids.filter((aid) => !existingAidsSet.has(aid)); - // 插入这些新条目 - const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')"); - newAids.forEach((aid) => insertStmt.run(aid)); + // 插入这些新条目 + const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')"); + newAids.forEach((aid) => insertStmt.run(aid)); } async function insertAidsToDB() { - if (shouldReadTextFile) { - await readFromText(); - } + if (shouldReadTextFile) { + await readFromText(); + } - const aidsInDB = db - .prepare("SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'") - .all() - .map((row) => row.aid) as number[]; + const aidsInDB = db + .prepare("SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'") + .all() + .map((row) => row.aid) as number[]; - const totalAids = aidsInDB.length; - let processedAids = 0; - const startTime = Date.now(); + const totalAids = aidsInDB.length; + let processedAids = 0; + const startTime = Date.now(); - const processAid = async (aid: number) => { - try { - const res = await getBiliBiliVideoInfo(aid, regions[processedAids % regions.length]); - if (res === null) { - updateAidStatus(aid, "failed"); - } else { - const rawData = JSON.parse(res); - if (rawData.code === 0) { - updateAidStatus(aid, "success", rawData.data.View.bvid, JSON.stringify(rawData.data)); - } else { - updateAidStatus(aid, "error", undefined, res); - } - } - } catch (error) { - console.error(`Error updating aid ${aid}: ${error}`); - updateAidStatus(aid, "failed"); - } finally { - processedAids++; - logProgress(aid, processedAids, totalAids, startTime); - } - }; + const processAid = async (aid: number) => { + try { + const res = await getBiliBiliVideoInfo(aid, regions[processedAids % regions.length]); + if (res === null) { + updateAidStatus(aid, "failed"); + } else { + const rawData = JSON.parse(res); + if (rawData.code === 0) { + updateAidStatus(aid, "success", rawData.data.View.bvid, JSON.stringify(rawData.data)); + } else { + updateAidStatus(aid, "error", undefined, res); + } + } + } catch (error) { + console.error(`Error updating aid ${aid}: ${error}`); + updateAidStatus(aid, "failed"); + } finally { + processedAids++; + logProgress(aid, processedAids, totalAids, startTime); + } + }; - const interval = setInterval(async () => { - if (aidsInDB.length === 0) { - clearInterval(interval); - console.log("All aids processed."); - return; - } - if (!isRateLimited()) { - const aid = aidsInDB.shift(); - if (aid !== undefined) { - requestQueue.push(Date.now()); - await processAid(aid); - } - } - }, 50); + const interval = setInterval(async () => { + if (aidsInDB.length === 0) { + clearInterval(interval); + console.log("All aids processed."); + return; + } + if (!isRateLimited()) { + const aid = aidsInDB.shift(); + if (aid !== undefined) { + requestQueue.push(Date.now()); + await processAid(aid); + } + } + }, 50); - console.log("Starting to process aids..."); + console.log("Starting to process aids..."); } function updateAidStatus(aid: number, status: string, bvid?: string, data?: string) { - const stmt = db.prepare(` + const stmt = db.prepare(` UPDATE bili_info_crawl SET status = ?, ${bvid ? "bvid = ?," : ""} @@ -145,31 +145,35 @@ function updateAidStatus(aid: number, status: string, bvid?: string, data?: stri timestamp = ? WHERE aid = ? `); - const params = [status, ...(bvid ? [bvid] : []), ...(data ? [data] : []), Date.now() / 1000, aid]; - stmt.run(...params); + const params = [status, ...(bvid ? [bvid] : []), ...(data ? [data] : []), Date.now() / 1000, aid]; + stmt.run(...params); } function logProgress(aid: number, processedAids: number, totalAids: number, startTime: number) { - const elapsedTime = Date.now() - startTime; - const elapsedSeconds = Math.floor(elapsedTime / 1000); - const elapsedMinutes = Math.floor(elapsedSeconds / 60); - const elapsedHours = Math.floor(elapsedMinutes / 60); + const elapsedTime = Date.now() - startTime; + const elapsedSeconds = Math.floor(elapsedTime / 1000); + const elapsedMinutes = Math.floor(elapsedSeconds / 60); + const elapsedHours = Math.floor(elapsedMinutes / 60); - const remainingAids = totalAids - processedAids; - const averageTimePerAid = elapsedTime / processedAids; - const eta = remainingAids * averageTimePerAid; - const etaSeconds = Math.floor(eta / 1000); - const etaMinutes = Math.floor(etaSeconds / 60); - const etaHours = Math.floor(etaMinutes / 60); + const remainingAids = totalAids - processedAids; + const averageTimePerAid = elapsedTime / processedAids; + const eta = remainingAids * averageTimePerAid; + const etaSeconds = Math.floor(eta / 1000); + const etaMinutes = Math.floor(etaSeconds / 60); + const etaHours = Math.floor(etaMinutes / 60); - const progress = `${processedAids}/${totalAids}, ${((processedAids / totalAids) * 100).toFixed( - 2 - )}%, elapsed ${elapsedHours.toString().padStart(2, "0")}:${(elapsedMinutes % 60).toString().padStart(2, "0")}:${( - elapsedSeconds % 60 - ) - .toString() - .padStart(2, "0")}, ETA ${etaHours}h${(etaMinutes % 60).toString().padStart(2, "0")}m`; - console.log(`Updated aid ${aid}, ${progress}`); + const progress = `${processedAids}/${totalAids}, ${ + ((processedAids / totalAids) * 100).toFixed( + 2, + ) + }%, elapsed ${elapsedHours.toString().padStart(2, "0")}:${(elapsedMinutes % 60).toString().padStart(2, "0")}:${ + ( + elapsedSeconds % 60 + ) + .toString() + .padStart(2, "0") + }, ETA ${etaHours}h${(etaMinutes % 60).toString().padStart(2, "0")}m`; + console.log(`Updated aid ${aid}, ${progress}`); } await setupLogging(); diff --git a/src/db/raw/videoInfo.ts b/src/db/raw/videoInfo.ts index dd0435f..54b3f32 100644 --- a/src/db/raw/videoInfo.ts +++ b/src/db/raw/videoInfo.ts @@ -1,53 +1,51 @@ export async function getBiliBiliVideoInfo(bvidORaid?: string | number, region: string = "hangzhou") { - const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined; - const aid = typeof bvidORaid === "number" ? bvidORaid : undefined; + const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined; + const aid = typeof bvidORaid === "number" ? bvidORaid : undefined; - const baseURL = "https://api.bilibili.com/x/web-interface/view/detail"; - const urlObject = new URL(baseURL); + const baseURL = "https://api.bilibili.com/x/web-interface/view/detail"; + const urlObject = new URL(baseURL); - if (aid) { - urlObject.searchParams.append("aid", aid.toString()); - const finalURL = urlObject.toString(); - return await proxyRequestWithRegion(finalURL, region); - } else if (bvid) { - urlObject.searchParams.append("bvid", bvid); - const finalURL = urlObject.toString(); - return await proxyRequestWithRegion(finalURL, region); - } else { - return null; - } + if (aid) { + urlObject.searchParams.append("aid", aid.toString()); + const finalURL = urlObject.toString(); + return await proxyRequestWithRegion(finalURL, region); + } else if (bvid) { + urlObject.searchParams.append("bvid", bvid); + const finalURL = urlObject.toString(); + return await proxyRequestWithRegion(finalURL, region); + } else { + return null; + } } async function proxyRequestWithRegion(url: string, region: string): Promise { - const td = new TextDecoder(); - const p = await new Deno.Command("aliyun", { - args: [ - "fc", - "POST", - `/2023-03-30/functions/proxy-${region}/invocations`, - "--qualifier", - "LATEST", - "--header", - "Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;", - "--body", - JSON.stringify({url: url}), - "--profile", - `CVSA-${region}`, - ], - }).output(); - try { - const out = td.decode(p.stdout); - const rawData = JSON.parse(out); - if (rawData.statusCode !== 200) { - console.error(`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`); - return null; - } - else { - return JSON.parse(rawData.body); - } - } - catch (e){ - console.error(`Error proxying request ${url} to ${region}: ${e}`); - return null; - } -} \ No newline at end of file + const td = new TextDecoder(); + const p = await new Deno.Command("aliyun", { + args: [ + "fc", + "POST", + `/2023-03-30/functions/proxy-${region}/invocations`, + "--qualifier", + "LATEST", + "--header", + "Content-Type=application/json;x-fc-invocation-type=Sync;x-fc-log-type=None;", + "--body", + JSON.stringify({ url: url }), + "--profile", + `CVSA-${region}`, + ], + }).output(); + try { + const out = td.decode(p.stdout); + const rawData = JSON.parse(out); + if (rawData.statusCode !== 200) { + console.error(`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`); + return null; + } else { + return JSON.parse(rawData.body); + } + } catch (e) { + console.error(`Error proxying request ${url} to ${region}: ${e}`); + return null; + } +} diff --git a/src/net/getLatestVideos.ts b/src/net/getLatestVideos.ts index 1b4b016..b6df59e 100644 --- a/src/net/getLatestVideos.ts +++ b/src/net/getLatestVideos.ts @@ -1,3 +1,35 @@ +import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; + +const API_URL = "https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=50&pn="; + +const requiredEnvVars = ["DB_HOST", "DB_NAME", "DB_USER", "DB_PASSWORD", "DB_PORT"]; + +const unsetVars = requiredEnvVars.filter((key) => !Deno.env.get(key)); + +if (unsetVars.length > 0) { + throw new Error(`Missing required environment variables: ${unsetVars.join(", ")}`); +} + +const databaseHost = Deno.env.get("DB_HOST")!; +const databaseName = Deno.env.get("DB_NAME")!; +const databaseUser = Deno.env.get("DB_USER")!; +const databasePassword = Deno.env.get("DB_PASSWORD")!; +const databasePort = Deno.env.get("DB_PORT")!; + +const postgresConfig = { + hostname: databaseHost, + port: parseInt(databasePort), + database: databaseName, + user: databaseUser, + password: databasePassword, +}; + +async function connectToPostgres() { + const client = new Client(postgresConfig); + await client.connect(); + return client; +} + export async function getLatestVideos() { - const baseURL = "" -} \ No newline at end of file + const client = await connectToPostgres(); +} diff --git a/static/favicon.ico b/static/favicon.ico new file mode 100644 index 0000000..1cfaaa2 Binary files /dev/null and b/static/favicon.ico differ diff --git a/static/logo.svg b/static/logo.svg new file mode 100644 index 0000000..ef2fbe4 --- /dev/null +++ b/static/logo.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/static/styles.css b/static/styles.css new file mode 100644 index 0000000..b5c61c9 --- /dev/null +++ b/static/styles.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/tailwind.config.ts b/tailwind.config.ts new file mode 100644 index 0000000..0c790d0 --- /dev/null +++ b/tailwind.config.ts @@ -0,0 +1,7 @@ +import { type Config } from "tailwindcss"; + +export default { + content: [ + "{routes,islands,components}/**/*.{ts,tsx,js,jsx}", + ], +} satisfies Config;