merge: branch 'main' into lab/pred
This commit is contained in:
commit
81847cc090
GPG Key ID:
56209E0CCD8420C6
@ -3,3 +3,4 @@ data
|
|||||||
*.svg
|
*.svg
|
||||||
*.txt
|
*.txt
|
||||||
*.md
|
*.md
|
||||||
|
*config*
|
||||||
@ -1,65 +0,0 @@
|
|||||||
# 项目重构方案
|
|
||||||
|
|
||||||
## 目标架构
|
|
||||||
采用monorepo结构管理三个独立部分：
|
|
||||||
1. `packages/crawler` - 现有爬虫功能
|
|
||||||
2. `packages/frontend` - 基于Astro的前端
|
|
||||||
3. `packages/backend` - 基于Hono的API后端
|
|
||||||
|
|
||||||
## 目录结构调整方案
|
|
||||||
|
|
||||||
### 新结构
|
|
||||||
```
|
|
||||||
.
|
|
||||||
├── packages/
|
|
||||||
│ ├── crawler/ # 爬虫组件
|
|
||||||
│ ├── frontend/ # Astro前端
|
|
||||||
│ ├── backend/ # Hono后端API
|
|
||||||
│ └── core/ # 共享代码(未来提取)
|
|
||||||
├── docs/ # 文档
|
|
||||||
├── scripts/ # 项目脚本
|
|
||||||
└── package.json # 根项目配置
|
|
||||||
```
|
|
||||||
|
|
||||||
### 具体迁移方案
|
|
||||||
|
|
||||||
#### 1. 爬虫部分(crawler)
|
|
||||||
保留以下目录/文件：
|
|
||||||
- `lib/` (除前端相关)
|
|
||||||
- `src/db/raw/`
|
|
||||||
- `src/filterWorker.ts`
|
|
||||||
- `src/worker.ts`
|
|
||||||
- `test/`
|
|
||||||
- `deno.json`
|
|
||||||
- `.gitignore`
|
|
||||||
|
|
||||||
需要移除：
|
|
||||||
- Fresh框架相关文件
|
|
||||||
- 前端组件(`components/`)
|
|
||||||
- 静态资源(`static/`)
|
|
||||||
|
|
||||||
#### 2. 前端部分(frontend)
|
|
||||||
全新创建Astro项目，不保留任何现有前端代码
|
|
||||||
|
|
||||||
#### 3. 后端部分(backend)
|
|
||||||
全新创建Hono项目
|
|
||||||
|
|
||||||
#### 4. 共享代码(core)
|
|
||||||
未来可从爬虫中提取以下内容到core package：
|
|
||||||
- 数据库相关：`lib/db/`
|
|
||||||
- 消息队列：`lib/mq/`
|
|
||||||
- 网络请求：`lib/net/`
|
|
||||||
- 工具函数：`lib/utils/`
|
|
||||||
|
|
||||||
## 重构步骤建议
|
|
||||||
|
|
||||||
1. 初始化monorepo结构
|
|
||||||
2. 迁移爬虫代码到`packages/crawler`
|
|
||||||
3. 创建新的Astro项目在`packages/frontend`
|
|
||||||
4. 创建新的Hono项目在`packages/backend`
|
|
||||||
5. 逐步提取共享代码到`packages/core`
|
|
||||||
|
|
||||||
## 注意事项
|
|
||||||
- 机器学习相关代码(`pred/`, `filter/`, `lab/`)保持现状
|
|
||||||
- 文档(`doc/`)可以迁移到`docs/`目录
|
|
||||||
- 需要更新CI/CD流程支持monorepo
|
|
||||||
16
deno.json
16
deno.json
@ -3,6 +3,20 @@
|
|||||||
"workspace": ["./packages/crawler", "./packages/frontend", "./packages/backend", "./packages/core"],
|
"workspace": ["./packages/crawler", "./packages/frontend", "./packages/backend", "./packages/core"],
|
||||||
"nodeModulesDir": "auto",
|
"nodeModulesDir": "auto",
|
||||||
"tasks": {
|
"tasks": {
|
||||||
"crawler": "deno task --filter 'crawler' all"
|
"crawler": "deno task --filter 'crawler' all",
|
||||||
|
"backend": "deno task --filter 'backend' start"
|
||||||
|
},
|
||||||
|
"fmt": {
|
||||||
|
"useTabs": true,
|
||||||
|
"lineWidth": 120,
|
||||||
|
"indentWidth": 4,
|
||||||
|
"semiColons": true,
|
||||||
|
"proseWrap": "always"
|
||||||
|
},
|
||||||
|
"imports": {
|
||||||
|
"@astrojs/node": "npm:@astrojs/node@^9.1.3",
|
||||||
|
"@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8",
|
||||||
|
"@core/db/": "./packages/core/db/",
|
||||||
|
"date-fns": "npm:date-fns@^4.1.0"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -9,12 +9,11 @@
|
|||||||
|
|
||||||
## Architecure
|
## Architecure
|
||||||
|
|
||||||
- [Overview](architecure/overview.md)
|
* [Overview](architecure/overview.md)
|
||||||
- [Database Structure](architecure/database-structure/README.md)
|
* [Database Structure](architecure/database-structure/README.md)
|
||||||
- [Type of Song](architecure/database-structure/type-of-song.md)
|
* [Type of Song](architecure/database-structure/type-of-song.md)
|
||||||
- [Message Queue](architecure/message-queue/README.md)
|
* [Message Queue](architecure/message-queue.md)
|
||||||
- [VideoTagsQueue](architecure/message-queue/videotagsqueue.md)
|
* [Artificial Intelligence](architecure/artificial-intelligence.md)
|
||||||
- [Artificial Intelligence](architecure/artificial-intelligence.md)
|
|
||||||
|
|
||||||
## API Doc
|
## API Doc
|
||||||
|
|
||||||
|
|||||||
@ -7,13 +7,23 @@ For a **song**, it must meet the following conditions to be included in CVSA:
|
|||||||
|
|
||||||
### Category 30
|
### Category 30
|
||||||
|
|
||||||
In principle, the songs featured in CVSA must be included in a video categorized under VOCALOID·UTAU (ID 30) that is
|
In principle, the songs must be featured in a video that is categorized under the VOCALOID·UTAU (ID 30) category in [Bilibili](https://en.wikipedia.org/wiki/Bilibili) in order to be observed by our [automation program](../architecure/overview.md#crawler). We welcome editors to manually add songs that have not been uploaded to bilibili / categorized under this category.
|
||||||
posted on Bilibili. In some special cases, this rule may not be enforced. 
|
|
||||||
|
|
||||||
### At Leats One Line of Chinese
|
#### NEWS
|
||||||
|
|
||||||
The lyrics of the song must contain at least one line in Chinese. This means that even if a voicebank that only supports
|
Recently, Bilibili seems to be offlining the sub-category. This means the VOCALOID·UTAU category can no longer be entered from the frontend, and producers can no longer upload videos to this category (instead, they can only choose the parent category "Music"). 
|
||||||
Chinese is used, if the lyrics of the song do not contain Chinese, it will not be included in the CVSA.
|
|
||||||
|
According to our experiments, Bilibili still retains the code logic of sub-categories in the backend, and newly published songs may still be in the VOCALOID·UTAU sub-category, and the related APIs can still work normally. However, there are [reports](https://www.bilibili.com/opus/1041223385394184199) that some of the new songs have been placed under the "Music General" sub-category.\
|
||||||
|
We are still waiting for Bilibili's follow-up actions, and in the future, we may adjust the scope of our automated program's crawling.
|
||||||
|
|
||||||
|
### At Leats One Line of Chinese / Chinese Virtual Singer
|
||||||
|
|
||||||
|
The lyrics of the song must contain at least one line in Chinese. Otherwise, if the lyrics of the song do not contain Chinese, it will only be included in the CVSA only if a Chinese virtual singer has been used.
|
||||||
|
|
||||||
|
We define a **Chinese virtual singer** as follows:
|
||||||
|
|
||||||
|
1. The singer primarily uses Chinese voicebank (i.e. the most widely used voickbank for the singer is Chinese)
|
||||||
|
2. The singer is operated by a company, organization, individual or group located in Mainland China, Hong Kong, Macau or Taiwan.
|
||||||
|
|
||||||
### Using Vocal Synthesizer
|
### Using Vocal Synthesizer
|
||||||
|
|
||||||
|
|||||||
@ -9,6 +9,10 @@ The AI systems we currently use are:
|
|||||||
Located at `/filter/` under project root dir, it classifies a video in the
|
Located at `/filter/` under project root dir, it classifies a video in the
|
||||||
[category 30](../about/scope-of-inclusion.md#category-30) into the following categories:
|
[category 30](../about/scope-of-inclusion.md#category-30) into the following categories:
|
||||||
|
|
||||||
- 0: Not related to Chinese vocal synthesis
|
* 0: Not related to Chinese vocal synthesis
|
||||||
- 1: A original song with Chinese vocal synthesis
|
* 1: A original song with Chinese vocal synthesis
|
||||||
- 2: A cover/remix song with Chinese vocal synthesis
|
* 2: A cover/remix song with Chinese vocal synthesis
|
||||||
|
|
||||||
|
### The Predictor
|
||||||
|
|
||||||
|
Located at `/pred/`under the project root dir, it predicts the future views of a video. This is a regression model that takes historical view trends of a video, other contextual information (such as the current time), and future time points to be predicted as feature inputs, and outputs the increment in the video's view count from "now" to the specified future time point.
|
||||||
|
|||||||
@ -5,8 +5,10 @@ CVSA uses [PostgreSQL](https://www.postgresql.org/) as our database.
|
|||||||
All public data of CVSA (excluding users' personal data) is stored in a database named `cvsa_main`, which contains the
|
All public data of CVSA (excluding users' personal data) is stored in a database named `cvsa_main`, which contains the
|
||||||
following tables:
|
following tables:
|
||||||
|
|
||||||
- songs: stores the main information of songs
|
* songs: stores the main information of songs
|
||||||
- bili\_user: stores snapshots of Bilibili user information
|
* bili\_user: stores snapshots of Bilibili user information
|
||||||
- all\_data: metadata of all videos in [category 30](../../about/scope-of-inclusion.md#category-30).
|
* all\_data: metadata of all videos in [category 30](../../about/scope-of-inclusion.md#category-30).
|
||||||
- labelling\_result: Contains label of videos in `all_data`tagged by our
|
* labelling\_result: Contains label of videos in `all_data`tagged by our [AI system](../artificial-intelligence.md#the-filter).
|
||||||
[AI system](../artificial-intelligence.md#the-filter).
|
* video\_snapshot: Statistical data of videos that are fetched regularly (e.g., number of views, etc.), we call this fetch process as "snapshot".
|
||||||
|
* snapshot\_schedule: The scheduling information for video snapshots.
|
||||||
|
|
||||||
|
|||||||
7
doc/en/architecure/message-queue.md
Normal file
7
doc/en/architecure/message-queue.md
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
# Message Queue
|
||||||
|
|
||||||
|
We rely on message queues to manage the various tasks that [the cralwer ](overview.md#crawler)needs to perform.
|
||||||
|
|
||||||
|
### Code Path
|
||||||
|
|
||||||
|
Currently, the code related to message queues are located at `lib/mq` and `src`.
|
||||||
@ -1 +0,0 @@
|
|||||||
# Message Queue
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
# VideoTagsQueue
|
|
||||||
|
|
||||||
### Jobs
|
|
||||||
|
|
||||||
The VideoTagsQueue contains two jobs: `getVideoTags`and `getVideosTags`. The former is used to fetch the tags of a
|
|
||||||
video, and the latter is responsible for scheduling the former.
|
|
||||||
|
|
||||||
### Return value
|
|
||||||
|
|
||||||
The return values across two jobs follows the following table:
|
|
||||||
|
|
||||||
<table><thead><tr><th width="168">Return Value</th><th>Description</th></tr></thead><tbody><tr><td>0</td><td>In <code>getVideoTags</code>: the tags was successfully fetched<br>In <code>getVideosTags</code>: all null-tags videos have a corresponding job successfully queued.</td></tr><tr><td>1</td><td>Used in <code>getVideoTags</code>: occured <code>fetch</code>error during the job</td></tr><tr><td>2</td><td>Used in <code>getVideoTags</code>: we've reached the rate limit set in NetScheduler</td></tr><tr><td>3</td><td>Used in <code>getVideoTags</code>: did't provide aid in the job data</td></tr><tr><td>4</td><td>Used in<code>getVideosTags</code>: There's no video with NULL as `tags`</td></tr><tr><td>1xx</td><td>Used in<code>getVideosTags</code>: the number of tasks in the queue has exceeded the limit, thus <code>getVideosTags</code> stops adding tasks. <code>xx</code> is the number of jobs added to the queue during execution.</td></tr></tbody></table>
|
|
||||||
@ -1,5 +1,4 @@
|
|||||||
---
|
---
|
||||||
icon: globe-pointer
|
|
||||||
layout:
|
layout:
|
||||||
title:
|
title:
|
||||||
visible: true
|
visible: true
|
||||||
@ -15,5 +14,14 @@ layout:
|
|||||||
|
|
||||||
# Overview
|
# Overview
|
||||||
|
|
||||||
Automation is the biggest highlight of CVSA's technical design. To achieve this, we use a message queue powered by
|
The whole CVSA system can be sperate into three different parts:
|
||||||
[BullMQ](https://bullmq.io/) to concurrently process various tasks in the data collection life cycle.
|
|
||||||
|
* Frontend
|
||||||
|
* API
|
||||||
|
* Crawler
|
||||||
|
|
||||||
|
The frontend is driven by [Astro](https://astro.build/) and is used to display the final CVSA page. The API is driven by [Hono](https://hono.dev) and is used to query the database and provide REST/GraphQL APIs that can be called by out website, applications, or third parties. The crawler is our automatic data collector, used to automatically collect new songs from bilibili, track their statistics, etc.
|
||||||
|
|
||||||
|
### Crawler
|
||||||
|
|
||||||
|
Automation is the biggest highlight of CVSA's technical design. To achieve this, we use a message queue powered by [BullMQ](https://bullmq.io/) to concurrently process various tasks in the data collection life cycle.
|
||||||
|
|||||||
@ -9,12 +9,12 @@
|
|||||||
|
|
||||||
## 技术架构 <a href="#architecture" id="architecture"></a>
|
## 技术架构 <a href="#architecture" id="architecture"></a>
|
||||||
|
|
||||||
- [概览](architecture/overview.md)
|
* [概览](architecture/overview.md)
|
||||||
- [数据库结构](architecture/database-structure/README.md)
|
* [数据库结构](architecture/database-structure/README.md)
|
||||||
- [歌曲类型](architecture/database-structure/type-of-song.md)
|
* [歌曲类型](architecture/database-structure/type-of-song.md)
|
||||||
- [人工智能](architecture/artificial-intelligence.md)
|
* [人工智能](architecture/artificial-intelligence.md)
|
||||||
- [消息队列](architecture/message-queue/README.md)
|
* [消息队列](architecture/message-queue/README.md)
|
||||||
- [VideoTagsQueue队列](architecture/message-queue/video-tags-queue.md)
|
* [LatestVideosQueue 队列](architecture/message-queue/latestvideosqueue-dui-lie.md)
|
||||||
|
|
||||||
## API 文档 <a href="#api-doc" id="api-doc"></a>
|
## API 文档 <a href="#api-doc" id="api-doc"></a>
|
||||||
|
|
||||||
|
|||||||
@ -4,7 +4,11 @@ CVSA 使用 [PostgreSQL](https://www.postgresql.org/) 作为数据库。
|
|||||||
|
|
||||||
CVSA 的所有公开数据（不包括用户的个人数据）都存储在名为 `cvsa_main` 的数据库中，该数据库包含以下表：
|
CVSA 的所有公开数据（不包括用户的个人数据）都存储在名为 `cvsa_main` 的数据库中，该数据库包含以下表：
|
||||||
|
|
||||||
- songs：存储歌曲的主要信息
|
* songs：存储歌曲的主要信息
|
||||||
- bili\_user：存储 Bilibili 用户信息快照
|
* bilibili\_user：存储 Bilibili 用户信息快照
|
||||||
- all\_data：[分区 30](../../about/scope-of-inclusion.md#vocaloiduatu-fen-qu) 中所有视频的元数据。
|
* bilibili\_metadata：[分区 30](../../about/scope-of-inclusion.md#vocaloiduatu-fen-qu) 中所有视频的元数据
|
||||||
- labelling\_result：包含由我们的 AI 系统 标记的 `all_data` 中视频的标签。
|
* labelling\_result：包含由我们的 AI 系统 标记的 `all_data` 中视频的标签。
|
||||||
|
* latest\_video\_snapshot：存储视频最新的快照
|
||||||
|
* video\_snapshot：存储视频的快照，包括特定时间下视频的统计信息（播放量、点赞数等）
|
||||||
|
* snapshot\_schedule：视频快照的规划信息，为辅助表
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1,2 @@
|
|||||||
|
# LatestVideosQueue 队列
|
||||||
|
|
||||||
@ -1,15 +0,0 @@
|
|||||||
---
|
|
||||||
description: 关于VideoTagsQueue队列的信息。
|
|
||||||
---
|
|
||||||
|
|
||||||
# VideoTagsQueue队列
|
|
||||||
|
|
||||||
### 任务
|
|
||||||
|
|
||||||
视频标签队列包含两个任务：`getVideoTags`和`getVideosTags`。前者用于获取视频的标签，后者负责调度前者。
|
|
||||||
|
|
||||||
### 返回值
|
|
||||||
|
|
||||||
两个任务的返回值遵循以下表格：
|
|
||||||
|
|
||||||
<table><thead><tr><th width="168">返回值</th><th>描述</th></tr></thead><tbody><tr><td>0</td><td>在 <code>getVideoTags</code> 中：标签成功获取<br>在 <code>getVideosTags</code> 中：所有无标签视频的相应任务已成功排队。</td></tr><tr><td>1</td><td>在 <code>getVideoTags</code> 中：任务期间发生 <code>fetch</code> 错误</td></tr><tr><td>2</td><td>在 <code>getVideoTags</code> 中：已达到 NetScheduler 设置的速率限制</td></tr><tr><td>3</td><td>在 <code>getVideoTags</code> 中：未在任务数据中提供帮助</td></tr><tr><td>4</td><td>在 <code>getVideosTags</code> 中：没有视频的 `tags` 为 NULL</td></tr><tr><td>1xx</td><td>在 <code>getVideosTags</code> 中：队列中的任务数量超过了限制，因此 <code>getVideosTags</code> 停止添加任务。<code>xx</code> 是在执行期间添加到队列的任务数量。</td></tr></tbody></table>
|
|
||||||
@ -1,5 +1,4 @@
|
|||||||
---
|
---
|
||||||
icon: globe-pointer
|
|
||||||
layout:
|
layout:
|
||||||
title:
|
title:
|
||||||
visible: true
|
visible: true
|
||||||
@ -15,4 +14,14 @@ layout:
|
|||||||
|
|
||||||
# 概览
|
# 概览
|
||||||
|
|
||||||
自动化是 CVSA 技术设计的最大亮点，为了实现自动化，我们使用BullMQ驱动的消息队列来并发处理数据采集生命周期中的各项任务。
|
整个CVSA项目分为三个组件：**crawler**, **frontend** 和 **backend。**
|
||||||
|
|
||||||
|
### **crawler**
|
||||||
|
|
||||||
|
位于项目目录`packages/crawler` 下，它负责以下工作：
|
||||||
|
|
||||||
|
* 抓取新的视频并收录作品
|
||||||
|
* 持续监控视频的播放量等统计信息
|
||||||
|
|
||||||
|
整个 crawler 由 BullMQ 消息队列驱动，使用 Redis 和 PostgreSQL 管理状态。
|
||||||
|
|
||||||
|
|||||||
20
packages/backend/database.ts
Normal file
20
packages/backend/database.ts
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
import { type Client, Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||||
|
import { postgresConfig } from "@core/db/pgConfig.ts";
|
||||||
|
import { createMiddleware } from "hono/factory";
|
||||||
|
|
||||||
|
const pool = new Pool(postgresConfig, 4);
|
||||||
|
|
||||||
|
export const db = pool;
|
||||||
|
|
||||||
|
export const dbMiddleware = createMiddleware(async (c, next) => {
|
||||||
|
const connection = await pool.connect();
|
||||||
|
c.set("db", connection);
|
||||||
|
await next();
|
||||||
|
connection.release();
|
||||||
|
});
|
||||||
|
|
||||||
|
declare module "hono" {
|
||||||
|
interface ContextVariableMap {
|
||||||
|
db: Client;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,17 @@
|
|||||||
|
{
|
||||||
|
"name": "@cvsa/backend",
|
||||||
|
"imports": {
|
||||||
|
"hono": "jsr:@hono/hono@^4.7.5",
|
||||||
|
"zod": "npm:zod",
|
||||||
|
"yup": "npm:yup"
|
||||||
|
},
|
||||||
|
"tasks": {
|
||||||
|
"dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts",
|
||||||
|
"start": "deno serve --env-file=.env --allow-env --allow-net --host 127.0.0.1 main.ts"
|
||||||
|
},
|
||||||
|
"compilerOptions": {
|
||||||
|
"jsx": "precompile",
|
||||||
|
"jsxImportSource": "hono/jsx"
|
||||||
|
},
|
||||||
|
"exports": "./main.ts"
|
||||||
|
}
|
||||||
20
packages/backend/main.ts
Normal file
20
packages/backend/main.ts
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
import { Hono } from "hono";
|
||||||
|
import { dbMiddleware } from "./database.ts";
|
||||||
|
import { rootHandler } from "./root.ts";
|
||||||
|
import { getSnapshotsHanlder } from "./snapshots.ts";
|
||||||
|
|
||||||
|
export const app = new Hono();
|
||||||
|
|
||||||
|
app.use('/video/*', dbMiddleware);
|
||||||
|
|
||||||
|
app.get("/", ...rootHandler);
|
||||||
|
|
||||||
|
app.get('/video/:id/snapshots', ...getSnapshotsHanlder);
|
||||||
|
|
||||||
|
const fetch = app.fetch;
|
||||||
|
|
||||||
|
export default {
|
||||||
|
fetch,
|
||||||
|
} satisfies Deno.ServeDefaultExport;
|
||||||
|
|
||||||
|
export const VERSION = "0.2.0";
|
||||||
31
packages/backend/root.ts
Normal file
31
packages/backend/root.ts
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
import { getSingerForBirthday, pickSinger, pickSpecialSinger, type Singer } from "./singers.ts";
|
||||||
|
import { VERSION } from "./main.ts";
|
||||||
|
import { createHandlers } from "./utils.ts";
|
||||||
|
|
||||||
|
export const rootHandler = createHandlers((c) => {
|
||||||
|
let singer: Singer | Singer[] | null = null;
|
||||||
|
const shouldShowSpecialSinger = Math.random() < 0.016;
|
||||||
|
if (getSingerForBirthday().length !== 0){
|
||||||
|
singer = getSingerForBirthday();
|
||||||
|
for (const s of singer) {
|
||||||
|
delete s.birthday;
|
||||||
|
s.message = `祝${s.name}生日快乐~`
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (shouldShowSpecialSinger) {
|
||||||
|
singer = pickSpecialSinger();
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
singer = pickSinger();
|
||||||
|
}
|
||||||
|
return c.json({
|
||||||
|
"project": {
|
||||||
|
"name": "中V档案馆",
|
||||||
|
"motto": "一起唱吧，心中的歌！"
|
||||||
|
},
|
||||||
|
"status": 200,
|
||||||
|
"version": VERSION,
|
||||||
|
"time": Date.now(),
|
||||||
|
"singer": singer
|
||||||
|
})
|
||||||
|
})
|
||||||
103
packages/backend/singers.ts
Normal file
103
packages/backend/singers.ts
Normal file
@ -0,0 +1,103 @@
|
|||||||
|
export const singers = [
|
||||||
|
{
|
||||||
|
"name": "洛天依",
|
||||||
|
"color": "#66CCFF",
|
||||||
|
"birthday": "0712",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "言和",
|
||||||
|
"color": "#00FFCC",
|
||||||
|
"birthday": "0711",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "乐正绫",
|
||||||
|
"color": "#EE0000",
|
||||||
|
"birthday": "0412",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "乐正龙牙",
|
||||||
|
"color": "#006666",
|
||||||
|
"birthday": "1002",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "徵羽摩柯",
|
||||||
|
"color": "#0080FF",
|
||||||
|
"birthday": "1210",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "墨清弦",
|
||||||
|
"color": "#FFFF00",
|
||||||
|
"birthday": "0520",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "星尘",
|
||||||
|
"color": "#9999FF",
|
||||||
|
"birthday": "0812",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "心华",
|
||||||
|
"color": "#EE82EE",
|
||||||
|
"birthday": "0210",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "海伊",
|
||||||
|
"color": "#3399FF",
|
||||||
|
"birthday": "0722",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "苍穹",
|
||||||
|
"color": "#8BC0B5",
|
||||||
|
"birthday": "0520",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "赤羽",
|
||||||
|
"color": "#FF4004",
|
||||||
|
"birthday": "1126",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "诗岸",
|
||||||
|
"color": "#F6BE72",
|
||||||
|
"birthday": "0119",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "牧心",
|
||||||
|
"color": "#2A2859",
|
||||||
|
"birthday": "0807",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export interface Singer {
|
||||||
|
name: string;
|
||||||
|
color?: string;
|
||||||
|
birthday?: string;
|
||||||
|
message?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const specialSingers = [
|
||||||
|
{
|
||||||
|
"name": "雅音宫羽",
|
||||||
|
"message": "你是我最真模样，从来不曾遗忘。",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "初音未来",
|
||||||
|
"message": "初始之音，响彻未来!",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
export const pickSinger = () => {
|
||||||
|
const index = Math.floor(Math.random() * singers.length);
|
||||||
|
return singers[index];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const pickSpecialSinger = () => {
|
||||||
|
const index = Math.floor(Math.random() * specialSingers.length);
|
||||||
|
return specialSingers[index];
|
||||||
|
};
|
||||||
|
|
||||||
|
export const getSingerForBirthday = (): Singer[] => {
|
||||||
|
const today = new Date();
|
||||||
|
const month = String(today.getMonth() + 1).padStart(2, "0");
|
||||||
|
const day = String(today.getDate()).padStart(2, "0");
|
||||||
|
const datestring = `${month}${day}`;
|
||||||
|
return singers.filter((singer) => singer.birthday === datestring);
|
||||||
|
};
|
||||||
89
packages/backend/snapshots.ts
Normal file
89
packages/backend/snapshots.ts
Normal file
@ -0,0 +1,89 @@
|
|||||||
|
import type { Context } from "hono";
|
||||||
|
import { createHandlers } from "./utils.ts";
|
||||||
|
import type { BlankEnv, BlankInput } from "hono/types";
|
||||||
|
import { getVideoSnapshots, getVideoSnapshotsByBV } from "@core/db/videoSnapshot.ts";
|
||||||
|
import type { VideoSnapshotType } from "@core/db/schema.d.ts";
|
||||||
|
import { boolean, mixed, number, object, ValidationError } from "yup";
|
||||||
|
|
||||||
|
const SnapshotQueryParamsSchema = object({
|
||||||
|
ps: number().optional().positive(),
|
||||||
|
pn: number().optional().positive(),
|
||||||
|
offset: number().optional().positive(),
|
||||||
|
reverse: boolean().optional(),
|
||||||
|
});
|
||||||
|
|
||||||
|
const idSchema = mixed().test(
|
||||||
|
"is-valid-id",
|
||||||
|
'id must be a string starting with "av" followed by digits, or "BV" followed by 10 alphanumeric characters, or a positive integer',
|
||||||
|
(value) => {
|
||||||
|
if (typeof value === "number") {
|
||||||
|
return Number.isInteger(value) && value > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof value === "string") {
|
||||||
|
if (value.startsWith("av")) {
|
||||||
|
const digitsOnly = value.substring(2);
|
||||||
|
return /^\d+$/.test(digitsOnly) && digitsOnly.length > 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value.startsWith("BV")) {
|
||||||
|
const remainingChars = value.substring(2);
|
||||||
|
return /^[a-zA-Z0-9]{10}$/.test(remainingChars);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
type ContextType = Context<BlankEnv, "/video/:id/snapshots", BlankInput>;
|
||||||
|
export const getSnapshotsHanlder = createHandlers(async (c: ContextType) => {
|
||||||
|
const client = c.get("db");
|
||||||
|
|
||||||
|
try {
|
||||||
|
const idParam = await idSchema.validate(c.req.param("id"));
|
||||||
|
let videoId: number | string = idParam as string | number;
|
||||||
|
if (typeof videoId === "string" && videoId.startsWith("av")) {
|
||||||
|
videoId = videoId.slice(2);
|
||||||
|
}
|
||||||
|
const queryParams = await SnapshotQueryParamsSchema.validate(c.req.query());
|
||||||
|
const { ps, pn, offset, reverse = false } = queryParams;
|
||||||
|
|
||||||
|
let limit = 1000;
|
||||||
|
if (ps && ps > 1) {
|
||||||
|
limit = ps;
|
||||||
|
}
|
||||||
|
|
||||||
|
let pageOrOffset = 1;
|
||||||
|
let mode: "page" | "offset" = "page";
|
||||||
|
|
||||||
|
if (pn && pn > 1) {
|
||||||
|
pageOrOffset = pn;
|
||||||
|
mode = "page";
|
||||||
|
} else if (offset && offset > 1) {
|
||||||
|
pageOrOffset = offset;
|
||||||
|
mode = "offset";
|
||||||
|
}
|
||||||
|
|
||||||
|
let result: VideoSnapshotType[];
|
||||||
|
|
||||||
|
if (typeof videoId === "number") {
|
||||||
|
result = await getVideoSnapshots(client, videoId, limit, pageOrOffset, reverse, mode);
|
||||||
|
} else {
|
||||||
|
result = await getVideoSnapshotsByBV(client, videoId, limit, pageOrOffset, reverse, mode);
|
||||||
|
}
|
||||||
|
|
||||||
|
const rows = result.map((row) => ({
|
||||||
|
...row,
|
||||||
|
aid: Number(row.aid),
|
||||||
|
}));
|
||||||
|
|
||||||
|
return c.json(rows);
|
||||||
|
} catch (e) {
|
||||||
|
if (e instanceof ValidationError) {
|
||||||
|
return c.json({ message: "Invalid query parameters", errors: e.errors }, 400);
|
||||||
|
} else {
|
||||||
|
return c.json({ message: "Unhandled error", error: e }, 500);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
5
packages/backend/utils.ts
Normal file
5
packages/backend/utils.ts
Normal file
@ -0,0 +1,5 @@
|
|||||||
|
import { createFactory } from 'hono/factory'
|
||||||
|
|
||||||
|
const factory = createFactory();
|
||||||
|
|
||||||
|
export const createHandlers = factory.createHandlers;
|
||||||
@ -8,6 +8,7 @@ if (unsetVars.length > 0) {
|
|||||||
|
|
||||||
const databaseHost = Deno.env.get("DB_HOST")!;
|
const databaseHost = Deno.env.get("DB_HOST")!;
|
||||||
const databaseName = Deno.env.get("DB_NAME")!;
|
const databaseName = Deno.env.get("DB_NAME")!;
|
||||||
|
const databaseNameCred = Deno.env.get("DB_NAME_CRED")!;
|
||||||
const databaseUser = Deno.env.get("DB_USER")!;
|
const databaseUser = Deno.env.get("DB_USER")!;
|
||||||
const databasePassword = Deno.env.get("DB_PASSWORD")!;
|
const databasePassword = Deno.env.get("DB_PASSWORD")!;
|
||||||
const databasePort = Deno.env.get("DB_PORT")!;
|
const databasePort = Deno.env.get("DB_PORT")!;
|
||||||
@ -19,3 +20,11 @@ export const postgresConfig = {
|
|||||||
user: databaseUser,
|
user: databaseUser,
|
||||||
password: databasePassword,
|
password: databasePassword,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const postgresConfigCred = {
|
||||||
|
hostname: databaseHost,
|
||||||
|
port: parseInt(databasePort),
|
||||||
|
database: databaseNameCred,
|
||||||
|
user: databaseUser,
|
||||||
|
password: databasePassword,
|
||||||
|
};
|
||||||
55
packages/core/db/schema.d.ts
vendored
Normal file
55
packages/core/db/schema.d.ts
vendored
Normal file
@ -0,0 +1,55 @@
|
|||||||
|
export interface AllDataType {
|
||||||
|
id: number;
|
||||||
|
aid: bigint;
|
||||||
|
bvid: string | null;
|
||||||
|
description: string | null;
|
||||||
|
uid: number | null;
|
||||||
|
tags: string | null;
|
||||||
|
title: string | null;
|
||||||
|
published_at: string | null;
|
||||||
|
duration: number;
|
||||||
|
created_at: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface BiliUserType {
|
||||||
|
id: number;
|
||||||
|
uid: number;
|
||||||
|
username: string;
|
||||||
|
desc: string;
|
||||||
|
fans: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface VideoSnapshotType {
|
||||||
|
id: number;
|
||||||
|
created_at: string;
|
||||||
|
views: number;
|
||||||
|
coins: number;
|
||||||
|
likes: number;
|
||||||
|
favorites: number;
|
||||||
|
shares: number;
|
||||||
|
danmakus: number;
|
||||||
|
aid: bigint;
|
||||||
|
replies: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LatestSnapshotType {
|
||||||
|
aid: bigint;
|
||||||
|
time: number;
|
||||||
|
views: number;
|
||||||
|
danmakus: number;
|
||||||
|
replies: number;
|
||||||
|
likes: number;
|
||||||
|
coins: number;
|
||||||
|
shares: number;
|
||||||
|
favorites: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SnapshotScheduleType {
|
||||||
|
id: number;
|
||||||
|
aid: bigint;
|
||||||
|
type?: string;
|
||||||
|
created_at: string;
|
||||||
|
started_at?: string;
|
||||||
|
finished_at?: string;
|
||||||
|
status: string;
|
||||||
|
}
|
||||||
33
packages/core/db/videoSnapshot.ts
Normal file
33
packages/core/db/videoSnapshot.ts
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||||
|
import { VideoSnapshotType } from "@core/db/schema.d.ts";
|
||||||
|
|
||||||
|
export async function getVideoSnapshots(client: Client, aid: number, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
|
||||||
|
const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||||
|
const order = reverse ? 'ASC' : 'DESC';
|
||||||
|
const query = `
|
||||||
|
SELECT *
|
||||||
|
FROM video_snapshot
|
||||||
|
WHERE aid = $1
|
||||||
|
ORDER BY created_at ${order}
|
||||||
|
LIMIT $2
|
||||||
|
OFFSET $3
|
||||||
|
`;
|
||||||
|
const queryResult = await client.queryObject<VideoSnapshotType>(query, [aid, limit, offset]);
|
||||||
|
return queryResult.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getVideoSnapshotsByBV(client: Client, bv: string, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
|
||||||
|
const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||||
|
const order = reverse ? 'ASC' : 'DESC';
|
||||||
|
const query = `
|
||||||
|
SELECT vs.*
|
||||||
|
FROM video_snapshot vs
|
||||||
|
JOIN bilibili_metadata bm ON vs.aid = bm.aid
|
||||||
|
WHERE bm.bvid = $1
|
||||||
|
ORDER BY vs.created_at ${order}
|
||||||
|
LIMIT $2
|
||||||
|
OFFSET $3
|
||||||
|
`
|
||||||
|
const queryResult = await client.queryObject<VideoSnapshotType>(query, [bv, limit, offset]);
|
||||||
|
return queryResult.rows;
|
||||||
|
}
|
||||||
@ -1,5 +1,5 @@
|
|||||||
import { Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
import { Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||||
import { postgresConfig } from "db/pgConfig.ts";
|
import { postgresConfig } from "@core/db/pgConfig.ts";
|
||||||
|
|
||||||
const pool = new Pool(postgresConfig, 12);
|
const pool = new Pool(postgresConfig, 12);
|
||||||
|
|
||||||
|
|||||||
@ -28,7 +28,7 @@ export async function refreshSnapshotWindowCounts(client: Client, redisClient: R
|
|||||||
WHERE started_at >= NOW() AND status = 'pending' AND started_at <= NOW() + INTERVAL '10 days'
|
WHERE started_at >= NOW() AND status = 'pending' AND started_at <= NOW() + INTERVAL '10 days'
|
||||||
GROUP BY 1
|
GROUP BY 1
|
||||||
ORDER BY window_start
|
ORDER BY window_start
|
||||||
`
|
`;
|
||||||
|
|
||||||
await redisClient.del(REDIS_KEY);
|
await redisClient.del(REDIS_KEY);
|
||||||
|
|
||||||
@ -36,7 +36,7 @@ export async function refreshSnapshotWindowCounts(client: Client, redisClient: R
|
|||||||
|
|
||||||
for (const row of result.rows) {
|
for (const row of result.rows) {
|
||||||
const targetOffset = Math.floor((row.window_start.getTime() - startTime) / (5 * MINUTE));
|
const targetOffset = Math.floor((row.window_start.getTime() - startTime) / (5 * MINUTE));
|
||||||
const offset = (currentWindow + targetOffset);
|
const offset = currentWindow + targetOffset;
|
||||||
if (offset >= 0) {
|
if (offset >= 0) {
|
||||||
await redisClient.hset(REDIS_KEY, offset.toString(), Number(row.count));
|
await redisClient.hset(REDIS_KEY, offset.toString(), Number(row.count));
|
||||||
}
|
}
|
||||||
@ -186,7 +186,13 @@ export async function getSnapshotScheduleCountWithinRange(client: Client, start:
|
|||||||
* @param aid The aid of the video.
|
* @param aid The aid of the video.
|
||||||
* @param targetTime Scheduled time for snapshot. (Timestamp in milliseconds)
|
* @param targetTime Scheduled time for snapshot. (Timestamp in milliseconds)
|
||||||
*/
|
*/
|
||||||
export async function scheduleSnapshot(client: Client, aid: number, type: string, targetTime: number, force: boolean = false) {
|
export async function scheduleSnapshot(
|
||||||
|
client: Client,
|
||||||
|
aid: number,
|
||||||
|
type: string,
|
||||||
|
targetTime: number,
|
||||||
|
force: boolean = false,
|
||||||
|
) {
|
||||||
if (await videoHasActiveSchedule(client, aid) && !force) return;
|
if (await videoHasActiveSchedule(client, aid) && !force) return;
|
||||||
let adjustedTime = new Date(targetTime);
|
let adjustedTime = new Date(targetTime);
|
||||||
if (type !== "milestone" && type !== "new") {
|
if (type !== "milestone" && type !== "new") {
|
||||||
@ -199,7 +205,13 @@ export async function scheduleSnapshot(client: Client, aid: number, type: string
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function bulkScheduleSnapshot(client: Client, aids: number[], type: string, targetTime: number, force: boolean = false) {
|
export async function bulkScheduleSnapshot(
|
||||||
|
client: Client,
|
||||||
|
aids: number[],
|
||||||
|
type: string,
|
||||||
|
targetTime: number,
|
||||||
|
force: boolean = false,
|
||||||
|
) {
|
||||||
for (const aid of aids) {
|
for (const aid of aids) {
|
||||||
await scheduleSnapshot(client, aid, type, targetTime, force);
|
await scheduleSnapshot(client, aid, type, targetTime, force);
|
||||||
}
|
}
|
||||||
@ -253,7 +265,6 @@ export async function adjustSnapshotTime(
|
|||||||
return expectedStartTime;
|
return expectedStartTime;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
export async function getSnapshotsInNextSecond(client: Client) {
|
export async function getSnapshotsInNextSecond(client: Client) {
|
||||||
const query = `
|
const query = `
|
||||||
SELECT *
|
SELECT *
|
||||||
|
|||||||
@ -39,12 +39,5 @@
|
|||||||
"onnxruntime": "npm:onnxruntime-node@1.19.2",
|
"onnxruntime": "npm:onnxruntime-node@1.19.2",
|
||||||
"chalk": "npm:chalk"
|
"chalk": "npm:chalk"
|
||||||
},
|
},
|
||||||
"fmt": {
|
|
||||||
"useTabs": true,
|
|
||||||
"lineWidth": 120,
|
|
||||||
"indentWidth": 4,
|
|
||||||
"semiColons": true,
|
|
||||||
"proseWrap": "always"
|
|
||||||
},
|
|
||||||
"exports": "./main.ts"
|
"exports": "./main.ts"
|
||||||
}
|
}
|
||||||
@ -4,4 +4,4 @@
|
|||||||
// SO HERE'S A PLACHOLDER EXPORT FOR DENO:
|
// SO HERE'S A PLACHOLDER EXPORT FOR DENO:
|
||||||
export const DENO = "FUCK YOU DENO";
|
export const DENO = "FUCK YOU DENO";
|
||||||
// Oh, maybe export the version is a good idea
|
// Oh, maybe export the version is a good idea
|
||||||
export const VERSION = "1.0.13";
|
export const VERSION = "1.0.17";
|
||||||
|
|||||||
@ -7,6 +7,7 @@ import {
|
|||||||
bulkSetSnapshotStatus,
|
bulkSetSnapshotStatus,
|
||||||
findClosestSnapshot,
|
findClosestSnapshot,
|
||||||
findSnapshotBefore,
|
findSnapshotBefore,
|
||||||
|
getBulkSnapshotsInNextSecond,
|
||||||
getLatestSnapshot,
|
getLatestSnapshot,
|
||||||
getSnapshotsInNextSecond,
|
getSnapshotsInNextSecond,
|
||||||
getVideosWithoutActiveSnapshotSchedule,
|
getVideosWithoutActiveSnapshotSchedule,
|
||||||
@ -15,7 +16,6 @@ import {
|
|||||||
setSnapshotStatus,
|
setSnapshotStatus,
|
||||||
snapshotScheduleExists,
|
snapshotScheduleExists,
|
||||||
videoHasProcessingSchedule,
|
videoHasProcessingSchedule,
|
||||||
getBulkSnapshotsInNextSecond
|
|
||||||
} from "db/snapshotSchedule.ts";
|
} from "db/snapshotSchedule.ts";
|
||||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||||
import { HOUR, MINUTE, SECOND, WEEK } from "$std/datetime/constants.ts";
|
import { HOUR, MINUTE, SECOND, WEEK } from "$std/datetime/constants.ts";
|
||||||
@ -109,7 +109,7 @@ const log = (value: number, base: number = 10) => Math.log(value) / Math.log(bas
|
|||||||
* @param aid - aid of the video
|
* @param aid - aid of the video
|
||||||
* @returns ETA in hours
|
* @returns ETA in hours
|
||||||
*/
|
*/
|
||||||
const getAdjustedShortTermETA = async (client: Client, aid: number) => {
|
export const getAdjustedShortTermETA = async (client: Client, aid: number) => {
|
||||||
const latestSnapshot = await getLatestSnapshot(client, aid);
|
const latestSnapshot = await getLatestSnapshot(client, aid);
|
||||||
// Immediately dispatch a snapshot if there is no snapshot yet
|
// Immediately dispatch a snapshot if there is no snapshot yet
|
||||||
if (!latestSnapshot) return 0;
|
if (!latestSnapshot) return 0;
|
||||||
@ -117,7 +117,7 @@ const getAdjustedShortTermETA = async (client: Client, aid: number) => {
|
|||||||
if (!snapshotsEnough) return 0;
|
if (!snapshotsEnough) return 0;
|
||||||
|
|
||||||
const currentTimestamp = new Date().getTime();
|
const currentTimestamp = new Date().getTime();
|
||||||
const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR];
|
const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
|
||||||
const DELTA = 0.00001;
|
const DELTA = 0.00001;
|
||||||
let minETAHours = Infinity;
|
let minETAHours = Infinity;
|
||||||
|
|
||||||
@ -282,8 +282,7 @@ export const takeBulkSnapshotForVideosWorker = async (job: Job) => {
|
|||||||
}
|
}
|
||||||
logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");
|
logger.error(e as Error, "mq", "fn:takeBulkSnapshotForVideosWorker");
|
||||||
await bulkSetSnapshotStatus(client, ids, "failed");
|
await bulkSetSnapshotStatus(client, ids, "failed");
|
||||||
}
|
} finally {
|
||||||
finally {
|
|
||||||
client.release();
|
client.release();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -373,7 +372,7 @@ export const scheduleCleanupWorker = async (_job: Job) => {
|
|||||||
SELECT id, aid, type
|
SELECT id, aid, type
|
||||||
FROM snapshot_schedule
|
FROM snapshot_schedule
|
||||||
WHERE status IN ('pending', 'processing')
|
WHERE status IN ('pending', 'processing')
|
||||||
AND started_at < NOW() - INTERVAL '5 minutes'
|
AND started_at < NOW() - INTERVAL '30 minutes'
|
||||||
`;
|
`;
|
||||||
const { rows } = await client.queryObject<{ id: bigint; aid: bigint; type: string }>(query);
|
const { rows } = await client.queryObject<{ id: bigint; aid: bigint; type: string }>(query);
|
||||||
if (rows.length === 0) return;
|
if (rows.length === 0) return;
|
||||||
|
|||||||
@ -24,10 +24,11 @@ export async function insertVideoInfo(client: Client, aid: number) {
|
|||||||
const title = data.View.title;
|
const title = data.View.title;
|
||||||
const published_at = formatTimestampToPsql(data.View.pubdate * SECOND + 8 * HOUR);
|
const published_at = formatTimestampToPsql(data.View.pubdate * SECOND + 8 * HOUR);
|
||||||
const duration = data.View.duration;
|
const duration = data.View.duration;
|
||||||
|
const cover = data.View.pic;
|
||||||
await client.queryObject(
|
await client.queryObject(
|
||||||
`INSERT INTO bilibili_metadata (aid, bvid, description, uid, tags, title, published_at, duration)
|
`INSERT INTO bilibili_metadata (aid, bvid, description, uid, tags, title, published_at, duration, cover_url)
|
||||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
|
||||||
[aid, bvid, desc, uid, tags, title, published_at, duration],
|
[aid, bvid, desc, uid, tags, title, published_at, duration, cover],
|
||||||
);
|
);
|
||||||
const userExists = await userExistsInBiliUsers(client, aid);
|
const userExists = await userExistsInBiliUsers(client, aid);
|
||||||
if (!userExists) {
|
if (!userExists) {
|
||||||
|
|||||||
3
packages/crawler/net/bilibili.d.ts
vendored
3
packages/crawler/net/bilibili.d.ts
vendored
@ -13,7 +13,6 @@ export type MediaListInfoResponse = BaseResponse<MediaListInfoData>;
|
|||||||
|
|
||||||
export type MediaListInfoData = MediaListInfoItem[];
|
export type MediaListInfoData = MediaListInfoItem[];
|
||||||
|
|
||||||
|
|
||||||
export interface MediaListInfoItem {
|
export interface MediaListInfoItem {
|
||||||
attr: number;
|
attr: number;
|
||||||
bvid: string;
|
bvid: string;
|
||||||
@ -26,7 +25,7 @@ export interface MediaListInfoItem {
|
|||||||
reply: number;
|
reply: number;
|
||||||
share: number;
|
share: number;
|
||||||
thumb_up: number;
|
thumb_up: number;
|
||||||
}
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
interface VideoInfoData {
|
interface VideoInfoData {
|
||||||
|
|||||||
@ -6,13 +6,13 @@ import { lockManager } from "mq/lockManager.ts";
|
|||||||
import { WorkerError } from "mq/schema.ts";
|
import { WorkerError } from "mq/schema.ts";
|
||||||
import { getVideoInfoWorker } from "mq/exec/getLatestVideos.ts";
|
import { getVideoInfoWorker } from "mq/exec/getLatestVideos.ts";
|
||||||
import {
|
import {
|
||||||
|
bulkSnapshotTickWorker,
|
||||||
collectMilestoneSnapshotsWorker,
|
collectMilestoneSnapshotsWorker,
|
||||||
regularSnapshotsWorker,
|
regularSnapshotsWorker,
|
||||||
snapshotTickWorker,
|
|
||||||
takeSnapshotForVideoWorker,
|
|
||||||
scheduleCleanupWorker,
|
scheduleCleanupWorker,
|
||||||
|
snapshotTickWorker,
|
||||||
takeBulkSnapshotForVideosWorker,
|
takeBulkSnapshotForVideosWorker,
|
||||||
bulkSnapshotTickWorker
|
takeSnapshotForVideoWorker,
|
||||||
} from "mq/exec/snapshotTick.ts";
|
} from "mq/exec/snapshotTick.ts";
|
||||||
|
|
||||||
Deno.addSignalListener("SIGINT", async () => {
|
Deno.addSignalListener("SIGINT", async () => {
|
||||||
|
|||||||
24
packages/frontend/.gitignore
vendored
Normal file
24
packages/frontend/.gitignore
vendored
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
# build output
|
||||||
|
dist/
|
||||||
|
|
||||||
|
# generated types
|
||||||
|
.astro/
|
||||||
|
|
||||||
|
# dependencies
|
||||||
|
node_modules/
|
||||||
|
|
||||||
|
# logs
|
||||||
|
npm-debug.log*
|
||||||
|
yarn-debug.log*
|
||||||
|
yarn-error.log*
|
||||||
|
pnpm-debug.log*
|
||||||
|
|
||||||
|
# environment variables
|
||||||
|
.env
|
||||||
|
.env.production
|
||||||
|
|
||||||
|
# macOS-specific files
|
||||||
|
.DS_Store
|
||||||
|
|
||||||
|
# jetbrains setting folder
|
||||||
|
.idea/
|
||||||
24
packages/frontend/astro.config.mjs
Normal file
24
packages/frontend/astro.config.mjs
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
// @ts-check
|
||||||
|
import { defineConfig } from "astro/config";
|
||||||
|
import tailwind from "@astrojs/tailwind";
|
||||||
|
|
||||||
|
// https://astro.build/config
|
||||||
|
import tsconfigPaths from "vite-tsconfig-paths";
|
||||||
|
import node from "@astrojs/node";
|
||||||
|
import svelte from "@astrojs/svelte";
|
||||||
|
|
||||||
|
export default defineConfig({
|
||||||
|
output: "server",
|
||||||
|
adapter: node({
|
||||||
|
mode: "standalone",
|
||||||
|
}),
|
||||||
|
integrations: [tailwind(), svelte()],
|
||||||
|
vite: {
|
||||||
|
server: {
|
||||||
|
fs: {
|
||||||
|
allow: [".", "../../"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
plugins: [tsconfigPaths()],
|
||||||
|
},
|
||||||
|
});
|
||||||
@ -0,0 +1,8 @@
|
|||||||
|
{
|
||||||
|
"name": "@cvsa/frontend",
|
||||||
|
"tasks": {
|
||||||
|
"preview": "bun run astro preview",
|
||||||
|
"build": "bun run astro build"
|
||||||
|
},
|
||||||
|
"exports": "./main.ts"
|
||||||
|
}
|
||||||
1
packages/frontend/main.ts
Normal file
1
packages/frontend/main.ts
Normal file
@ -0,0 +1 @@
|
|||||||
|
export const VERSION = "1.2.6";
|
||||||
23
packages/frontend/package.json
Normal file
23
packages/frontend/package.json
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"name": "frontend",
|
||||||
|
"type": "module",
|
||||||
|
"version": "0.0.1",
|
||||||
|
"scripts": {
|
||||||
|
"dev": "astro dev",
|
||||||
|
"build": "astro build",
|
||||||
|
"preview": "astro preview",
|
||||||
|
"astro": "astro"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@astrojs/tailwind": "^6.0.2",
|
||||||
|
"astro": "^5.5.5",
|
||||||
|
"autoprefixer": "^10.4.21",
|
||||||
|
"pg": "^8.11.11",
|
||||||
|
"postcss": "^8.5.3",
|
||||||
|
"tailwindcss": "^3.0.24",
|
||||||
|
"vite-tsconfig-paths": "^5.1.4"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/pg": "^8.11.11"
|
||||||
|
}
|
||||||
|
}
|
||||||
9
packages/frontend/public/favicon.svg
Normal file
9
packages/frontend/public/favicon.svg
Normal file
@ -0,0 +1,9 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 128 128">
|
||||||
|
<path d="M50.4 78.5a75.1 75.1 0 0 0-28.5 6.9l24.2-65.7c.7-2 1.9-3.2 3.4-3.2h29c1.5 0 2.7 1.2 3.4 3.2l24.2 65.7s-11.6-7-28.5-7L67 45.5c-.4-1.7-1.6-2.8-2.9-2.8-1.3 0-2.5 1.1-2.9 2.7L50.4 78.5Zm-1.1 28.2Zm-4.2-20.2c-2 6.6-.6 15.8 4.2 20.2a17.5 17.5 0 0 1 .2-.7 5.5 5.5 0 0 1 5.7-4.5c2.8.1 4.3 1.5 4.7 4.7.2 1.1.2 2.3.2 3.5v.4c0 2.7.7 5.2 2.2 7.4a13 13 0 0 0 5.7 4.9v-.3l-.2-.3c-1.8-5.6-.5-9.5 4.4-12.8l1.5-1a73 73 0 0 0 3.2-2.2 16 16 0 0 0 6.8-11.4c.3-2 .1-4-.6-6l-.8.6-1.6 1a37 37 0 0 1-22.4 2.7c-5-.7-9.7-2-13.2-6.2Z" />
|
||||||
|
<style>
|
||||||
|
path { fill: #000; }
|
||||||
|
@media (prefers-color-scheme: dark) {
|
||||||
|
path { fill: #FFF; }
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 749 B |
7
packages/frontend/src/assets/TitleBar Mobile Dark.svg
Normal file
7
packages/frontend/src/assets/TitleBar Mobile Dark.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 6.1 KiB |
7
packages/frontend/src/assets/TitleBar Mobile Light.svg
Normal file
7
packages/frontend/src/assets/TitleBar Mobile Light.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 6.1 KiB |
15
packages/frontend/src/assets/TitleBar-Dark.svg
Normal file
15
packages/frontend/src/assets/TitleBar-Dark.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
15
packages/frontend/src/assets/TitleBar-Light.svg
Normal file
15
packages/frontend/src/assets/TitleBar-Light.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
15
packages/frontend/src/assets/header-logo-dark.svg
Normal file
15
packages/frontend/src/assets/header-logo-dark.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
15
packages/frontend/src/assets/header-logo-light.svg
Normal file
15
packages/frontend/src/assets/header-logo-light.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
15
packages/frontend/src/assets/标题-浅色.svg
Normal file
15
packages/frontend/src/assets/标题-浅色.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
15
packages/frontend/src/assets/标题-深色.svg
Normal file
15
packages/frontend/src/assets/标题-深色.svg
Normal file
File diff suppressed because one or more lines are too long
|
After Width: | Height: | Size: 22 KiB |
12
packages/frontend/src/components/CloseIcon.svelte
Normal file
12
packages/frontend/src/components/CloseIcon.svelte
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
export let className = "";
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class={className}>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="26" height="26" viewBox="0 0 24 24">
|
||||||
|
<path
|
||||||
|
fill="currentColor"
|
||||||
|
d="M6.4 19L5 17.6l5.6-5.6L5 6.4L6.4 5l5.6 5.6L17.6 5L19 6.4L13.4 12l5.6 5.6l-1.4 1.4l-5.6-5.6z"
|
||||||
|
/>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
41
packages/frontend/src/components/DarkModeImage.svelte
Normal file
41
packages/frontend/src/components/DarkModeImage.svelte
Normal file
@ -0,0 +1,41 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { onMount } from "svelte";
|
||||||
|
import { fade } from "svelte/transition";
|
||||||
|
|
||||||
|
export let lightSrc: string;
|
||||||
|
export let darkSrc: string;
|
||||||
|
export let alt: string = "";
|
||||||
|
export let className: string = "";
|
||||||
|
|
||||||
|
let isDarkMode = false;
|
||||||
|
let currentSrc: string;
|
||||||
|
let opacity = 0;
|
||||||
|
|
||||||
|
onMount(() => {
|
||||||
|
const handleDarkModeChange = (event: MediaQueryListEvent) => {
|
||||||
|
isDarkMode = event.matches;
|
||||||
|
currentSrc = isDarkMode ? darkSrc : lightSrc;
|
||||||
|
opacity = 1;
|
||||||
|
};
|
||||||
|
|
||||||
|
const darkModeMediaQuery = window.matchMedia("(prefers-color-scheme: dark)");
|
||||||
|
isDarkMode = darkModeMediaQuery.matches;
|
||||||
|
currentSrc = isDarkMode ? darkSrc : lightSrc;
|
||||||
|
opacity = 1;
|
||||||
|
|
||||||
|
darkModeMediaQuery.addEventListener("change", handleDarkModeChange);
|
||||||
|
|
||||||
|
return () => {
|
||||||
|
darkModeMediaQuery.removeEventListener("change", handleDarkModeChange);
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
$: currentSrc = isDarkMode ? darkSrc : lightSrc;
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<img
|
||||||
|
src={currentSrc}
|
||||||
|
{alt}
|
||||||
|
class={className}
|
||||||
|
style={`opacity: ${opacity}`}
|
||||||
|
/>
|
||||||
20
packages/frontend/src/components/MenuIcon.svelte
Normal file
20
packages/frontend/src/components/MenuIcon.svelte
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
export let className = '';
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class={className}>
|
||||||
|
<svg width="28.000000" height="28.000000" viewBox="0 0 28 28" fill="none" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||||
|
<desc>
|
||||||
|
Created with Pixso.
|
||||||
|
</desc>
|
||||||
|
<defs>
|
||||||
|
<clipPath id="clip97_210">
|
||||||
|
<rect id="菜单按钮" width="28.000000" height="28.000000" fill="white" fill-opacity="0"/>
|
||||||
|
</clipPath>
|
||||||
|
</defs>
|
||||||
|
<g clip-path="url(#clip97_210)">
|
||||||
|
<path id="path" d="M4.66 21C4.33 21 4.05 20.88 3.83 20.66C3.61 20.44 3.5 20.16 3.5 19.83C3.49 19.5 3.61 19.22 3.83 19C4.06 18.77 4.33 18.66 4.66 18.66L23.33 18.66C23.66 18.66 23.94 18.77 24.16 19C24.38 19.22 24.5 19.5 24.5 19.83C24.49 20.16 24.38 20.44 24.16 20.66C23.94 20.88 23.66 21 23.33 21L4.66 21ZM4.66 15.16C4.33 15.16 4.05 15.05 3.83 14.83C3.61 14.6 3.5 14.32 3.5 14C3.49 13.67 3.61 13.39 3.83 13.16C4.06 12.94 4.33 12.83 4.66 12.83L23.33 12.83C23.66 12.83 23.94 12.94 24.16 13.16C24.38 13.39 24.5 13.67 24.5 14C24.49 14.32 24.38 14.6 24.16 14.83C23.94 15.05 23.66 15.16 23.33 15.16L4.66 15.16ZM4.66 9.33C4.33 9.33 4.05 9.22 3.83 8.99C3.61 8.77 3.5 8.49 3.5 8.16C3.49 7.83 3.61 7.56 3.83 7.33C4.06 7.11 4.33 7 4.66 7L23.33 7C23.66 7 23.94 7.11 24.16 7.33C24.38 7.56 24.5 7.83 24.5 8.16C24.49 8.49 24.38 8.77 24.16 8.99C23.94 9.22 23.66 9.33 23.33 9.33L4.66 9.33Z" fill="currentColor" fill-opacity="1.000000" fill-rule="nonzero"/>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
|
||||||
|
</div>
|
||||||
44
packages/frontend/src/components/SearchBox.svelte
Normal file
44
packages/frontend/src/components/SearchBox.svelte
Normal file
@ -0,0 +1,44 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
let inputBox: HTMLInputElement | null = null;
|
||||||
|
|
||||||
|
export function changeFocusState(target: boolean) {
|
||||||
|
if (!inputBox) return;
|
||||||
|
if (target) {
|
||||||
|
inputBox.focus();
|
||||||
|
} else {
|
||||||
|
inputBox.blur();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleKeydown(event: KeyboardEvent) {
|
||||||
|
if (event.key === "Enter") {
|
||||||
|
event.preventDefault();
|
||||||
|
const input = event.target as HTMLInputElement;
|
||||||
|
const value: string = input.value.trim();
|
||||||
|
if (!value) return;
|
||||||
|
window.location.href = `/song/${value}/info`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<!-- svelte-ignore a11y_autofocus -->
|
||||||
|
<div
|
||||||
|
class="absolute left-0 md:left-96 ml-4 w-[calc(100%-5rem)] md:w-[calc(100%-40rem)] 2xl:max-w-[50rem] 2xl:left-1/2 2xl:-translate-x-1/2 inline-flex items-center h-full"
|
||||||
|
>
|
||||||
|
<input
|
||||||
|
bind:this={inputBox}
|
||||||
|
type="search"
|
||||||
|
placeholder="搜索"
|
||||||
|
class="top-0 w-full h-10 px-4 rounded-lg bg-white/80 dark:bg-zinc-800/70
|
||||||
|
backdrop-blur-lg border border-zinc-300 dark:border-zinc-600 focus:border-zinc-400 duration-200 transition-colors focus:outline-none"
|
||||||
|
on:keydown={handleKeydown}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<style>
|
||||||
|
[type="search"]::-webkit-search-cancel-button,
|
||||||
|
[type="search"]::-webkit-search-decoration {
|
||||||
|
-webkit-appearance: none;
|
||||||
|
appearance: none;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
12
packages/frontend/src/components/SearchIcon.svelte
Normal file
12
packages/frontend/src/components/SearchIcon.svelte
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
export let className = '';
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class={className}>
|
||||||
|
<svg xmlns="http://www.w3.org/2000/svg" width="26" height="26" viewBox="0 0 24 24">
|
||||||
|
<path
|
||||||
|
fill="currentColor"
|
||||||
|
d="m19.6 21l-6.3-6.3q-.75.6-1.725.95T9.5 16q-2.725 0-4.612-1.888T3 9.5t1.888-4.612T9.5 3t4.613 1.888T16 9.5q0 1.1-.35 2.075T14.7 13.3l6.3 6.3zM9.5 14q1.875 0 3.188-1.312T14 9.5t-1.312-3.187T9.5 5T6.313 6.313T5 9.5t1.313 3.188T9.5 14"
|
||||||
|
></path>
|
||||||
|
</svg>
|
||||||
|
</div>
|
||||||
30
packages/frontend/src/components/TitleBar.astro
Normal file
30
packages/frontend/src/components/TitleBar.astro
Normal file
@ -0,0 +1,30 @@
|
|||||||
|
---
|
||||||
|
import astroLogoLight from "@assets/标题-浅色.svg";
|
||||||
|
import astroLogoDark from "@assets/标题-深色.svg";
|
||||||
|
import DarkModeImage from "@components/DarkModeImage.svelte";
|
||||||
|
import SearchBox from "@components/SearchBox.svelte";
|
||||||
|
import TitleBarMobile from "@components/TitleBarMobile.svelte";
|
||||||
|
---
|
||||||
|
|
||||||
|
<div class="hidden md:block fixed top-0 left-0 w-full h-28 bg-white/80 dark:bg-zinc-900/70 backdrop-blur-lg z-50">
|
||||||
|
<div class="w-[305px] ml-8 inline-flex h-full items-center">
|
||||||
|
<a href="/">
|
||||||
|
<DarkModeImage
|
||||||
|
lightSrc={astroLogoLight.src}
|
||||||
|
darkSrc={astroLogoDark.src}
|
||||||
|
alt="Logo"
|
||||||
|
className="w-[305px] h-24 inline-block"
|
||||||
|
client:load
|
||||||
|
/>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
<SearchBox client:load />
|
||||||
|
|
||||||
|
<div
|
||||||
|
class="inline-flex right-12 absolute gap-4 h-full text-xl dark:text-[#C6DCF2] font-medium items-center w-48 justify-end"
|
||||||
|
>
|
||||||
|
<a href="/about" class="hover:dark:text-[#B1C5DA]">关于</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<TitleBarMobile client:load />
|
||||||
47
packages/frontend/src/components/TitleBarMobile.svelte
Normal file
47
packages/frontend/src/components/TitleBarMobile.svelte
Normal file
@ -0,0 +1,47 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import logoMobileDark from "@assets/TitleBar Mobile Dark.svg";
|
||||||
|
import logoMobileLight from "@assets/TitleBar Mobile Light.svg";
|
||||||
|
import SearchBox from "./SearchBox.svelte";
|
||||||
|
import SearchIcon from "./SearchIcon.svelte";
|
||||||
|
import MenuIcon from "./MenuIcon.svelte";
|
||||||
|
import DarkModeImage from "./DarkModeImage.svelte";
|
||||||
|
import CloseIcon from "./CloseIcon.svelte";
|
||||||
|
|
||||||
|
let searchBox: SearchBox | null = null;
|
||||||
|
let showSearchBox = false;
|
||||||
|
|
||||||
|
$: if (showSearchBox && searchBox) {
|
||||||
|
searchBox.changeFocusState(true);
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<div class="md:hidden fixed top-0 left-0 w-full h-16 bg-white/80 dark:bg-zinc-800/70 backdrop-blur-lg z-50">
|
||||||
|
{#if !showSearchBox}
|
||||||
|
<button class="inline-block ml-4 mt-4 dark:text-white">
|
||||||
|
<MenuIcon />
|
||||||
|
</button>
|
||||||
|
<div class="ml-8 inline-flex h-full items-center">
|
||||||
|
<a href="/">
|
||||||
|
<DarkModeImage
|
||||||
|
lightSrc={logoMobileLight.src}
|
||||||
|
darkSrc={logoMobileDark.src}
|
||||||
|
alt="Logo"
|
||||||
|
className="w-24 h-8 translate-y-[2px]"
|
||||||
|
/>
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
{#if showSearchBox}
|
||||||
|
<SearchBox bind:this={searchBox} />
|
||||||
|
{/if}
|
||||||
|
<button
|
||||||
|
class="inline-flex absolute right-0 h-full items-center mr-4"
|
||||||
|
onclick={() => (showSearchBox = !showSearchBox)}
|
||||||
|
>
|
||||||
|
{#if showSearchBox}
|
||||||
|
<CloseIcon />
|
||||||
|
{:else}
|
||||||
|
<SearchIcon />
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
10
packages/frontend/src/components/Welcome.astro
Normal file
10
packages/frontend/src/components/Welcome.astro
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
import TitleBar from "@components/TitleBar.astro";
|
||||||
|
---
|
||||||
|
|
||||||
|
<TitleBar/>
|
||||||
|
|
||||||
|
<main class="flex flex-col items-center justify-center min-h-screen gap-8">
|
||||||
|
<h1 class="text-4xl font-bold text-center">正在施工中……</h1>
|
||||||
|
<p>在搜索栏输入BV号或AV号，可以查询目前数据库收集到的信息~</p>
|
||||||
|
</main>
|
||||||
60
packages/frontend/src/content/about.md
Normal file
60
packages/frontend/src/content/about.md
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
# 关于「中V档案馆」
|
||||||
|
|
||||||
|
「中V档案馆」是一个旨在收录与展示「中文歌声合成作品」及有关信息的网站。
|
||||||
|
|
||||||
|
## 创建背景与关联工作
|
||||||
|
|
||||||
|
纵观整个互联网，对于「中文歌声合成」或「中文虚拟歌手」（常简称为中V或VC）相关信息进行较为系统、全面地整理收集的主要有以下几个网站：
|
||||||
|
|
||||||
|
- [萌娘百科](https://zh.moegirl.org.cn/):
|
||||||
|
收录了大量中V歌曲及歌姬的信息，呈现形式为传统维基（基于[MediaWiki](https://www.mediawiki.org/)）。
|
||||||
|
- [VCPedia](https://vcpedia.cn/):
|
||||||
|
由原萌娘百科中文歌声合成编辑团队的部分成员搭建，专属于中文歌声合成相关内容的信息集成站点[^1]，呈现形式为传统维基（基于[MediaWiki](https://www.mediawiki.org/)）。
|
||||||
|
- [VocaDB](https://vocadb.net/): 一个围绕 Vocaloid、UTAU 和其他歌声合成器的协作数据库，其中包含艺术家、唱片、PV
|
||||||
|
等[^2]，其中包含大量中文歌声合成作品。
|
||||||
|
- [天钿Daily](https://tdd.bunnyxt.com/)：一个VC相关数据交流与分享的网站。致力于VC相关数据交流，定期抓取VC相关数据，选取有意义的纬度展示。[^3]
|
||||||
|
|
||||||
|
上述网站中，或多或少存在一些不足，例如：
|
||||||
|
|
||||||
|
- 萌娘百科、VCPedia受限于传统维基，绝大多数内容依赖人工编辑。
|
||||||
|
- VocaDB基于结构化数据库构建，由此可以依赖程序生成一些信息，但**条目收录**仍然完全依赖人工完成。
|
||||||
|
- VocaDB主要专注于元数据展示，少有关于歌曲、作者等的描述性的文字，也缺乏描述性的背景信息。
|
||||||
|
- 天钿Daily只展示歌曲的统计数据及历史趋势，没有关于歌曲其它信息的收集。
|
||||||
|
|
||||||
|
因此，**中V档案馆**吸取前人经验，克服上述网站的不足，希望做到：
|
||||||
|
|
||||||
|
- 歌曲收录（指发现歌曲并创建条目）的完全自动化
|
||||||
|
- 歌曲元信息提取的高度自动化
|
||||||
|
- 歌曲统计数据收集的完全自动化
|
||||||
|
- 在程序辅助的同时欢迎并鼓励贡献者参与编辑（主要为描述性内容）或纠错
|
||||||
|
- 在适当的许可声明下，引用来自上述源的数据，使内容更加全面、丰富。
|
||||||
|
|
||||||
|
## 技术架构
|
||||||
|
|
||||||
|
参见[CVSA文档](https://cvsa.gitbook.io/)。
|
||||||
|
|
||||||
|
## 开放许可
|
||||||
|
|
||||||
|
受本文以[CC BY-NC-SA 4.0协议](https://creativecommons.org/licenses/by-nc-sa/4.0/)提供。
|
||||||
|
|
||||||
|
### 数据库
|
||||||
|
|
||||||
|
中V档案馆使用[PostgreSQL](https://postgresql.org)作为数据库，我们承诺定期导出数据库转储 (dump)
|
||||||
|
文件并公开，其内容遵从以下协议或条款：
|
||||||
|
|
||||||
|
- 数据库中的事实性数据，根据适用法律，不构成受版权保护的内容。中V档案馆放弃一切可能的权利（[CC0 1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/)）。
|
||||||
|
- 对于数据库中有原创性的内容（如贡献者编辑的描述性内容），如无例外，以[CC BY 4.0协议](https://creativecommons.org/licenses/by/4.0/)提供。
|
||||||
|
- 对于引用、摘编或改编自萌娘百科、VCPedia的内容，以与原始协议(CC BY-NC-SA 3.0
|
||||||
|
CN)兼容的协议[CC BY-NC-SA 4.0协议](https://creativecommons.org/licenses/by-nc-sa/4.0/)提供，并注明原始协议 。
|
||||||
|
> 根据原始协议第四条第2项内容，CC BY-NC-SA 4.0协议为与原始协议具有相同授权要素的后续版本（“可适用的协议”）。
|
||||||
|
- 中V档案馆文档使用[CC BY 4.0协议](https://creativecommons.org/licenses/by/4.0/)。
|
||||||
|
|
||||||
|
### 软件代码
|
||||||
|
|
||||||
|
用于构建中V档案馆的软件代码在[AGPL 3.0](https://www.gnu.org/licenses/agpl-3.0.html)许可证下公开，参见[LICENSE](./LICENSE)
|
||||||
|
|
||||||
|
[^1]: 引用自[VCPedia](https://vcpedia.cn/%E9%A6%96%E9%A1%B5)，于[知识共享 署名-非商业性使用-相同方式共享 3.0中国大陆 (CC BY-NC-SA 3.0 CN) 许可协议](https://creativecommons.org/licenses/by-nc-sa/3.0/cn/)下提供。
|
||||||
|
|
||||||
|
[^2]: 翻译自[VocaDB](https://vocadb.net/)，于[CC BY 4.0协议](https://creativecommons.org/licenses/by/4.0/)下提供。
|
||||||
|
|
||||||
|
[^3]: 引用自[关于 - 天钿Daily](https://tdd.bunnyxt.com/about)
|
||||||
15
packages/frontend/src/layouts/Layout.astro
Normal file
15
packages/frontend/src/layouts/Layout.astro
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
---
|
||||||
|
import "../styles/global.css";
|
||||||
|
---
|
||||||
|
|
||||||
|
<!doctype html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
|
<title>CVSA 前端</title>
|
||||||
|
</head>
|
||||||
|
<body class="dark:bg-zinc-900 dark:text-zinc-100">
|
||||||
|
<slot />
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
10
packages/frontend/src/pages/404.astro
Normal file
10
packages/frontend/src/pages/404.astro
Normal file
@ -0,0 +1,10 @@
|
|||||||
|
---
|
||||||
|
import Layout from '@layouts/Layout.astro';
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout>
|
||||||
|
<main class="flex flex-col items-center justify-center min-h-screen gap-8">
|
||||||
|
<h1 class="text-9xl font-thin">404</h1>
|
||||||
|
<p class="text-xl font-medium">咦……页面去哪里了(ﾟДﾟ≡ﾟдﾟ)!?</p>
|
||||||
|
</main>
|
||||||
|
</Layout>
|
||||||
15
packages/frontend/src/pages/about.astro
Normal file
15
packages/frontend/src/pages/about.astro
Normal file
@ -0,0 +1,15 @@
|
|||||||
|
---
|
||||||
|
import TitleBar from "@components/TitleBar.astro";
|
||||||
|
import Layout from '@layouts/Layout.astro';
|
||||||
|
import {Content as AboutContent} from '../content/about.md';
|
||||||
|
import "../styles/content.css";
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout>
|
||||||
|
<TitleBar/>
|
||||||
|
<main class="flex flex-col items-center min-h-screen gap-8 mt-36 relative z-0">
|
||||||
|
<div class="lg:w-1/2 content">
|
||||||
|
<AboutContent/>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
</Layout>
|
||||||
8
packages/frontend/src/pages/index.astro
Normal file
8
packages/frontend/src/pages/index.astro
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
---
|
||||||
|
import Welcome from '@components/Welcome.astro';
|
||||||
|
import Layout from '@layouts/Layout.astro';
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout>
|
||||||
|
<Welcome />
|
||||||
|
</Layout>
|
||||||
200
packages/frontend/src/pages/song/[id]/info.astro
Normal file
200
packages/frontend/src/pages/song/[id]/info.astro
Normal file
@ -0,0 +1,200 @@
|
|||||||
|
---
|
||||||
|
import Layout from "@layouts/Layout.astro";
|
||||||
|
import TitleBar from "@components/TitleBar.astro";
|
||||||
|
import pg from "pg";
|
||||||
|
import { format } from 'date-fns';
|
||||||
|
import { zhCN } from 'date-fns/locale';
|
||||||
|
|
||||||
|
const databaseHost = process.env.DB_HOST
|
||||||
|
const databaseName = process.env.DB_NAME
|
||||||
|
const databaseUser = process.env.DB_USER
|
||||||
|
const databasePassword = process.env.DB_PASSWORD
|
||||||
|
const databasePort = process.env.DB_PORT
|
||||||
|
|
||||||
|
const postgresConfig = {
|
||||||
|
hostname: databaseHost,
|
||||||
|
port: parseInt(databasePort!),
|
||||||
|
database: databaseName,
|
||||||
|
user: databaseUser,
|
||||||
|
password: databasePassword,
|
||||||
|
};
|
||||||
|
|
||||||
|
// 路由参数
|
||||||
|
const { id } = Astro.params;
|
||||||
|
const { Client } = pg;
|
||||||
|
const client = new Client(postgresConfig);
|
||||||
|
await client.connect();
|
||||||
|
|
||||||
|
// 数据库查询函数
|
||||||
|
async function getVideoMetadata(aid: number) {
|
||||||
|
const res = await client.query("SELECT * FROM bilibili_metadata WHERE aid = $1", [aid]);
|
||||||
|
if (res.rows.length <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const row = res.rows[0];
|
||||||
|
if (row) {
|
||||||
|
return row;
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getVideoSnapshots(aid: number) {
|
||||||
|
const res = await client.query("SELECT * FROM video_snapshot WHERE aid = $1 ORDER BY created_at DESC", [
|
||||||
|
aid,
|
||||||
|
]);
|
||||||
|
if (res.rows.length <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
return res.rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getAidFromBV(bv: string) {
|
||||||
|
const res = await client.query("SELECT aid FROM bilibili_metadata WHERE bvid = $1", [bv]);
|
||||||
|
if (res.rows.length <= 0) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const row = res.rows[0];
|
||||||
|
if (row && row.aid) {
|
||||||
|
return Number(row.aid);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getVideoAid(id: string) {
|
||||||
|
if (id.startsWith("av")) {
|
||||||
|
return parseInt(id.slice(2));
|
||||||
|
} else if (id.startsWith("BV")) {
|
||||||
|
return getAidFromBV(id);
|
||||||
|
}
|
||||||
|
return parseInt(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
// 获取数据
|
||||||
|
if (!id) {
|
||||||
|
Astro.response.status = 404;
|
||||||
|
client.end();
|
||||||
|
return new Response(null, { status: 404 });
|
||||||
|
}
|
||||||
|
const aid = await getVideoAid(id);
|
||||||
|
if (!aid || isNaN(aid)) {
|
||||||
|
Astro.response.status = 404;
|
||||||
|
client.end();
|
||||||
|
return new Response(null, { status: 404 });
|
||||||
|
}
|
||||||
|
const videoInfo = await getVideoMetadata(aid);
|
||||||
|
const snapshots = await getVideoSnapshots(aid);
|
||||||
|
client.end();
|
||||||
|
|
||||||
|
interface Snapshot {
|
||||||
|
created_at: Date;
|
||||||
|
views: number;
|
||||||
|
danmakus: number;
|
||||||
|
replies: number;
|
||||||
|
coins: number;
|
||||||
|
likes: number;
|
||||||
|
favorites: number;
|
||||||
|
shares: number;
|
||||||
|
id: number;
|
||||||
|
}
|
||||||
|
---
|
||||||
|
|
||||||
|
<Layout>
|
||||||
|
<TitleBar />
|
||||||
|
<main class="flex flex-col items-center min-h-screen gap-8 mt-36 relative z-0">
|
||||||
|
<div class="max-w-4xl mx-auto rounded-lg p-6">
|
||||||
|
<h1 class="text-2xl font-bold mb-4">视频信息: <a href={`https://www.bilibili.com/video/av${aid}`} class="underline">av{aid}</a></h1>
|
||||||
|
|
||||||
|
<div class="mb-6 p-4 rounded-lg">
|
||||||
|
<h2 class="text-xl font-semibold mb-8">基本信息</h2>
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
<table class="table-auto w-full">
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">ID</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.id}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">AID</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.aid}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">BVID</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.bvid}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">标题</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.title}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">描述</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.description}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">UID</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.uid}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">标签</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.tags}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">发布时间</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.published_at ? format(new Date(videoInfo.published_at), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN }) : '-'}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">时长 (秒)</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.duration}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">创建时间</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.created_at ? format(new Date(videoInfo.created_at), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN }) : '-'}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2 font-bold">封面</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{videoInfo?.cover_url ? videoInfo.cover_url : '-'}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="p-4 rounded-lg">
|
||||||
|
<h2 class="text-xl font-semibold mb-4">播放量历史数据</h2>
|
||||||
|
{snapshots && snapshots.length > 0 ? (
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
<table class="table-auto w-full">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">创建时间</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">观看</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">硬币</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">点赞</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">收藏</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">分享</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">弹幕</th>
|
||||||
|
<th class="border dark:border-zinc-500 px-4 py-2">评论</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{snapshots.map((snapshot: Snapshot) => (
|
||||||
|
<tr>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{format(new Date(snapshot.created_at), 'yyyy-MM-dd HH:mm:ss', { locale: zhCN })}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.views}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.coins}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.likes}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.favorites}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.shares}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.danmakus}</td>
|
||||||
|
<td class="border dark:border-zinc-500 px-4 py-2">{snapshot.replies}</td>
|
||||||
|
</tr>
|
||||||
|
))}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<p>暂无历史数据。</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</main>
|
||||||
|
</Layout>
|
||||||
76
packages/frontend/src/styles/content.css
Normal file
76
packages/frontend/src/styles/content.css
Normal file
@ -0,0 +1,76 @@
|
|||||||
|
@tailwind base;
|
||||||
|
@tailwind components;
|
||||||
|
@tailwind utilities;
|
||||||
|
|
||||||
|
.content {
|
||||||
|
@apply text-gray-800 dark:text-zinc-100;
|
||||||
|
|
||||||
|
h1,
|
||||||
|
h2,
|
||||||
|
h3,
|
||||||
|
h4 {
|
||||||
|
@apply font-bold text-gray-900 dark:text-white my-4;
|
||||||
|
}
|
||||||
|
|
||||||
|
h1 {
|
||||||
|
@apply text-3xl;
|
||||||
|
}
|
||||||
|
|
||||||
|
h2 {
|
||||||
|
@apply text-2xl;
|
||||||
|
}
|
||||||
|
|
||||||
|
h3 {
|
||||||
|
@apply text-xl;
|
||||||
|
}
|
||||||
|
|
||||||
|
h4 {
|
||||||
|
@apply text-lg;
|
||||||
|
}
|
||||||
|
|
||||||
|
p {
|
||||||
|
@apply my-4;
|
||||||
|
}
|
||||||
|
|
||||||
|
a {
|
||||||
|
@apply text-blue-500 hover:text-blue-700 dark:hover:text-blue-400 underline;
|
||||||
|
}
|
||||||
|
|
||||||
|
ul,
|
||||||
|
ol {
|
||||||
|
@apply list-disc list-inside my-4;
|
||||||
|
}
|
||||||
|
|
||||||
|
li {
|
||||||
|
@apply my-2;
|
||||||
|
}
|
||||||
|
|
||||||
|
blockquote {
|
||||||
|
@apply border-l-4 border-gray-300 pl-4 italic my-4;
|
||||||
|
}
|
||||||
|
|
||||||
|
code {
|
||||||
|
@apply bg-gray-100 text-gray-800 rounded px-1 duration-300;
|
||||||
|
}
|
||||||
|
|
||||||
|
pre {
|
||||||
|
@apply bg-gray-100 p-4 rounded overflow-x-auto my-4 duration-300 h-0;
|
||||||
|
}
|
||||||
|
|
||||||
|
table {
|
||||||
|
@apply w-full border-collapse my-4;
|
||||||
|
}
|
||||||
|
|
||||||
|
th,
|
||||||
|
td {
|
||||||
|
@apply border border-gray-300 p-2;
|
||||||
|
}
|
||||||
|
|
||||||
|
th {
|
||||||
|
@apply bg-gray-200 font-bold;
|
||||||
|
}
|
||||||
|
ul li p,
|
||||||
|
ol li p {
|
||||||
|
@apply inline;
|
||||||
|
}
|
||||||
|
}
|
||||||
3
packages/frontend/src/styles/global.css
Normal file
3
packages/frontend/src/styles/global.css
Normal file
@ -0,0 +1,3 @@
|
|||||||
|
@tailwind base;
|
||||||
|
@tailwind components;
|
||||||
|
@tailwind utilities;
|
||||||
8
packages/frontend/tailwind.config.js
Normal file
8
packages/frontend/tailwind.config.js
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
/** @type {import('tailwindcss').Config} */
|
||||||
|
module.exports = {
|
||||||
|
content: ["./src/**/*.{astro,html,js,jsx,md,mdx,svelte,ts,tsx,vue}"],
|
||||||
|
theme: {
|
||||||
|
extend: {},
|
||||||
|
},
|
||||||
|
plugins: [],
|
||||||
|
};
|
||||||
16
packages/frontend/tsconfig.json
Normal file
16
packages/frontend/tsconfig.json
Normal file
@ -0,0 +1,16 @@
|
|||||||
|
{
|
||||||
|
"extends": "astro/tsconfigs/strict",
|
||||||
|
"include": [".astro/types.d.ts", "**/*"],
|
||||||
|
"exclude": ["dist"],
|
||||||
|
"compilerOptions": {
|
||||||
|
"baseUrl": ".",
|
||||||
|
"paths": {
|
||||||
|
"@components/*": ["src/components/*"],
|
||||||
|
"@layouts/*": ["src/layouts/*"],
|
||||||
|
"@utils/*": ["src/utils/*"],
|
||||||
|
"@assets/*": ["src/assets/*"],
|
||||||
|
"@styles": ["src/styles/*"],
|
||||||
|
"@core/*": ["../core/*"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -14,14 +14,20 @@ const db = new Database(DATABASE_PATH, { int64: true });
|
|||||||
// 设置日志
|
// 设置日志
|
||||||
async function setupLogging() {
|
async function setupLogging() {
|
||||||
await ensureDir(LOG_DIR);
|
await ensureDir(LOG_DIR);
|
||||||
const logStream = await Deno.open(LOG_FILE, { write: true, create: true, append: true });
|
const logStream = await Deno.open(LOG_FILE, {
|
||||||
|
write: true,
|
||||||
|
create: true,
|
||||||
|
append: true,
|
||||||
|
});
|
||||||
|
|
||||||
const redirectConsole =
|
const redirectConsole =
|
||||||
// deno-lint-ignore no-explicit-any
|
// deno-lint-ignore no-explicit-any
|
||||||
(originalConsole: (...args: any[]) => void) =>
|
(originalConsole: (...args: any[]) => void) =>
|
||||||
// deno-lint-ignore no-explicit-any
|
// deno-lint-ignore no-explicit-any
|
||||||
(...args: any[]) => {
|
(...args: any[]) => {
|
||||||
const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
const message = args.map((
|
||||||
|
arg,
|
||||||
|
) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
||||||
originalConsole(message);
|
originalConsole(message);
|
||||||
logStream.write(new TextEncoder().encode(message + "\n"));
|
logStream.write(new TextEncoder().encode(message + "\n"));
|
||||||
};
|
};
|
||||||
@ -38,14 +44,17 @@ interface Metadata {
|
|||||||
|
|
||||||
// 获取最后一次更新的时间
|
// 获取最后一次更新的时间
|
||||||
function getLastUpdate(): Date {
|
function getLastUpdate(): Date {
|
||||||
const result = db.prepare("SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'").get() as Metadata;
|
const result = db.prepare(
|
||||||
|
"SELECT value FROM metadata WHERE key = 'fetchAid-lastUpdate'",
|
||||||
|
).get() as Metadata;
|
||||||
return result ? new Date(result.value as string) : new Date(0);
|
return result ? new Date(result.value as string) : new Date(0);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 更新最后更新时间
|
// 更新最后更新时间
|
||||||
function updateLastUpdate() {
|
function updateLastUpdate() {
|
||||||
const now = new Date().toISOString();
|
const now = new Date().toISOString();
|
||||||
db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'").run(now);
|
db.prepare("UPDATE metadata SET value = ? WHERE key = 'fetchAid-lastUpdate'")
|
||||||
|
.run(now);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 辅助函数：获取数据
|
// 辅助函数：获取数据
|
||||||
@ -66,7 +75,9 @@ async function fetchData(pn: number, retries = MAX_RETRIES): Promise<any> {
|
|||||||
|
|
||||||
// 插入 aid 到数据库
|
// 插入 aid 到数据库
|
||||||
function insertAid(aid: number) {
|
function insertAid(aid: number) {
|
||||||
db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')").run(aid);
|
db.prepare(
|
||||||
|
"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
|
||||||
|
).run(aid);
|
||||||
}
|
}
|
||||||
|
|
||||||
// 主函数
|
// 主函数
|
||||||
|
|||||||
@ -5,7 +5,16 @@ import { ensureDir } from "https://deno.land/std@0.113.0/fs/mod.ts";
|
|||||||
|
|
||||||
const aidPath = "./data/2025010104_c30_aids.txt";
|
const aidPath = "./data/2025010104_c30_aids.txt";
|
||||||
const db = new Database("./data/main.db", { int64: true });
|
const db = new Database("./data/main.db", { int64: true });
|
||||||
const regions = ["shanghai", "hangzhou", "qingdao", "beijing", "zhangjiakou", "chengdu", "shenzhen", "hohhot"];
|
const regions = [
|
||||||
|
"shanghai",
|
||||||
|
"hangzhou",
|
||||||
|
"qingdao",
|
||||||
|
"beijing",
|
||||||
|
"zhangjiakou",
|
||||||
|
"chengdu",
|
||||||
|
"shenzhen",
|
||||||
|
"hohhot",
|
||||||
|
];
|
||||||
const logDir = "./logs/bili-info-crawl";
|
const logDir = "./logs/bili-info-crawl";
|
||||||
const logFile = path.join(logDir, `run-${Date.now() / 1000}.log`);
|
const logFile = path.join(logDir, `run-${Date.now() / 1000}.log`);
|
||||||
const shouldReadTextFile = false;
|
const shouldReadTextFile = false;
|
||||||
@ -26,14 +35,20 @@ const requestQueue: number[] = [];
|
|||||||
|
|
||||||
async function setupLogging() {
|
async function setupLogging() {
|
||||||
await ensureDir(logDir);
|
await ensureDir(logDir);
|
||||||
const logStream = await Deno.open(logFile, { write: true, create: true, append: true });
|
const logStream = await Deno.open(logFile, {
|
||||||
|
write: true,
|
||||||
|
create: true,
|
||||||
|
append: true,
|
||||||
|
});
|
||||||
|
|
||||||
const redirectConsole =
|
const redirectConsole =
|
||||||
// deno-lint-ignore no-explicit-any
|
// deno-lint-ignore no-explicit-any
|
||||||
(originalConsole: (...args: any[]) => void) =>
|
(originalConsole: (...args: any[]) => void) =>
|
||||||
// deno-lint-ignore no-explicit-any
|
// deno-lint-ignore no-explicit-any
|
||||||
(...args: any[]) => {
|
(...args: any[]) => {
|
||||||
const message = args.map((arg) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
const message = args.map((
|
||||||
|
arg,
|
||||||
|
) => (typeof arg === "object" ? JSON.stringify(arg) : arg)).join(" ");
|
||||||
originalConsole(message);
|
originalConsole(message);
|
||||||
logStream.write(new TextEncoder().encode(message + "\n"));
|
logStream.write(new TextEncoder().encode(message + "\n"));
|
||||||
};
|
};
|
||||||
@ -78,7 +93,9 @@ async function readFromText() {
|
|||||||
const newAids = aids.filter((aid) => !existingAidsSet.has(aid));
|
const newAids = aids.filter((aid) => !existingAidsSet.has(aid));
|
||||||
|
|
||||||
// 插入这些新条目
|
// 插入这些新条目
|
||||||
const insertStmt = db.prepare("INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')");
|
const insertStmt = db.prepare(
|
||||||
|
"INSERT OR IGNORE INTO bili_info_crawl (aid, status) VALUES (?, 'pending')",
|
||||||
|
);
|
||||||
newAids.forEach((aid) => insertStmt.run(aid));
|
newAids.forEach((aid) => insertStmt.run(aid));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -88,7 +105,9 @@ async function insertAidsToDB() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const aidsInDB = db
|
const aidsInDB = db
|
||||||
.prepare("SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'")
|
.prepare(
|
||||||
|
"SELECT aid FROM bili_info_crawl WHERE status = 'pending' OR status = 'failed'",
|
||||||
|
)
|
||||||
.all()
|
.all()
|
||||||
.map((row) => row.aid) as number[];
|
.map((row) => row.aid) as number[];
|
||||||
|
|
||||||
@ -98,13 +117,21 @@ async function insertAidsToDB() {
|
|||||||
|
|
||||||
const processAid = async (aid: number) => {
|
const processAid = async (aid: number) => {
|
||||||
try {
|
try {
|
||||||
const res = await getBiliBiliVideoInfo(aid, regions[processedAids % regions.length]);
|
const res = await getBiliBiliVideoInfo(
|
||||||
|
aid,
|
||||||
|
regions[processedAids % regions.length],
|
||||||
|
);
|
||||||
if (res === null) {
|
if (res === null) {
|
||||||
updateAidStatus(aid, "failed");
|
updateAidStatus(aid, "failed");
|
||||||
} else {
|
} else {
|
||||||
const rawData = JSON.parse(res);
|
const rawData = JSON.parse(res);
|
||||||
if (rawData.code === 0) {
|
if (rawData.code === 0) {
|
||||||
updateAidStatus(aid, "success", rawData.data.View.bvid, JSON.stringify(rawData.data));
|
updateAidStatus(
|
||||||
|
aid,
|
||||||
|
"success",
|
||||||
|
rawData.data.View.bvid,
|
||||||
|
JSON.stringify(rawData.data),
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
updateAidStatus(aid, "error", undefined, res);
|
updateAidStatus(aid, "error", undefined, res);
|
||||||
}
|
}
|
||||||
@ -136,7 +163,12 @@ async function insertAidsToDB() {
|
|||||||
console.log("Starting to process aids...");
|
console.log("Starting to process aids...");
|
||||||
}
|
}
|
||||||
|
|
||||||
function updateAidStatus(aid: number, status: string, bvid?: string, data?: string) {
|
function updateAidStatus(
|
||||||
|
aid: number,
|
||||||
|
status: string,
|
||||||
|
bvid?: string,
|
||||||
|
data?: string,
|
||||||
|
) {
|
||||||
const stmt = db.prepare(`
|
const stmt = db.prepare(`
|
||||||
UPDATE bili_info_crawl
|
UPDATE bili_info_crawl
|
||||||
SET status = ?,
|
SET status = ?,
|
||||||
@ -145,11 +177,22 @@ function updateAidStatus(aid: number, status: string, bvid?: string, data?: stri
|
|||||||
timestamp = ?
|
timestamp = ?
|
||||||
WHERE aid = ?
|
WHERE aid = ?
|
||||||
`);
|
`);
|
||||||
const params = [status, ...(bvid ? [bvid] : []), ...(data ? [data] : []), Date.now() / 1000, aid];
|
const params = [
|
||||||
|
status,
|
||||||
|
...(bvid ? [bvid] : []),
|
||||||
|
...(data ? [data] : []),
|
||||||
|
Date.now() / 1000,
|
||||||
|
aid,
|
||||||
|
];
|
||||||
stmt.run(...params);
|
stmt.run(...params);
|
||||||
}
|
}
|
||||||
|
|
||||||
function logProgress(aid: number, processedAids: number, totalAids: number, startTime: number) {
|
function logProgress(
|
||||||
|
aid: number,
|
||||||
|
processedAids: number,
|
||||||
|
totalAids: number,
|
||||||
|
startTime: number,
|
||||||
|
) {
|
||||||
const elapsedTime = Date.now() - startTime;
|
const elapsedTime = Date.now() - startTime;
|
||||||
const elapsedSeconds = Math.floor(elapsedTime / 1000);
|
const elapsedSeconds = Math.floor(elapsedTime / 1000);
|
||||||
const elapsedMinutes = Math.floor(elapsedSeconds / 60);
|
const elapsedMinutes = Math.floor(elapsedSeconds / 60);
|
||||||
|
|||||||
@ -1,4 +1,7 @@
|
|||||||
export async function getBiliBiliVideoInfo(bvidORaid?: string | number, region: string = "hangzhou") {
|
export async function getBiliBiliVideoInfo(
|
||||||
|
bvidORaid?: string | number,
|
||||||
|
region: string = "hangzhou",
|
||||||
|
) {
|
||||||
const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined;
|
const bvid = typeof bvidORaid === "string" ? bvidORaid : undefined;
|
||||||
const aid = typeof bvidORaid === "number" ? bvidORaid : undefined;
|
const aid = typeof bvidORaid === "number" ? bvidORaid : undefined;
|
||||||
|
|
||||||
@ -18,7 +21,10 @@ export async function getBiliBiliVideoInfo(bvidORaid?: string | number, region:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function proxyRequestWithRegion(url: string, region: string): Promise<any | null> {
|
async function proxyRequestWithRegion(
|
||||||
|
url: string,
|
||||||
|
region: string,
|
||||||
|
): Promise<any | null> {
|
||||||
const td = new TextDecoder();
|
const td = new TextDecoder();
|
||||||
// aliyun configure set --access-key-id $ALIYUN_AK --access-key-secret $ALIYUN_SK --region cn-shenzhen --profile CVSA-shenzhen --mode AK
|
// aliyun configure set --access-key-id $ALIYUN_AK --access-key-secret $ALIYUN_SK --region cn-shenzhen --profile CVSA-shenzhen --mode AK
|
||||||
const p = await new Deno.Command("aliyun", {
|
const p = await new Deno.Command("aliyun", {
|
||||||
@ -40,7 +46,9 @@ async function proxyRequestWithRegion(url: string, region: string): Promise<any
|
|||||||
const out = td.decode(p.stdout);
|
const out = td.decode(p.stdout);
|
||||||
const rawData = JSON.parse(out);
|
const rawData = JSON.parse(out);
|
||||||
if (rawData.statusCode !== 200) {
|
if (rawData.statusCode !== 200) {
|
||||||
console.error(`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`);
|
console.error(
|
||||||
|
`Error proxying request ${url} to ${region} , statusCode: ${rawData.statusCode}`,
|
||||||
|
);
|
||||||
return null;
|
return null;
|
||||||
} else {
|
} else {
|
||||||
return JSON.parse(rawData.body);
|
return JSON.parse(rawData.body);
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user