Compare commits
No commits in common. "f4d08e944ab530d4e3debbbb36c72544ff4bf10a" and "69fb3604b16d9764f88245938e0516cf8ad8aaa2" have entirely different histories.
f4d08e944a
...
69fb3604b1
@ -14,11 +14,6 @@
|
||||
<excludeFolder url="file://$MODULE_DIR$/logs" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/model" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/src/db" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.idea" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.vscode" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/.zed" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/packages/frontend/.astro" />
|
||||
<excludeFolder url="file://$MODULE_DIR$/scripts" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
|
||||
@ -1,6 +0,0 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="DenoSettings">
|
||||
<option name="useDenoValue" value="ENABLE" />
|
||||
</component>
|
||||
</project>
|
||||
@ -16,6 +16,7 @@
|
||||
"imports": {
|
||||
"@astrojs/node": "npm:@astrojs/node@^9.1.3",
|
||||
"@astrojs/svelte": "npm:@astrojs/svelte@^7.0.8",
|
||||
"@core/db/": "./packages/core/db/",
|
||||
"date-fns": "npm:date-fns@^4.1.0"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,21 +1,21 @@
|
||||
# Table of contents
|
||||
|
||||
* [Welcome](README.md)
|
||||
- [Welcome](README.md)
|
||||
|
||||
## About
|
||||
|
||||
* [About CVSA Project](about/this-project.md)
|
||||
* [Scope of Inclusion](about/scope-of-inclusion.md)
|
||||
- [About CVSA Project](about/this-project.md)
|
||||
- [Scope of Inclusion](about/scope-of-inclusion.md)
|
||||
|
||||
## Architecure
|
||||
|
||||
* [Overview](architecure/overview.md)
|
||||
* [Crawler](architecure/crawler.md)
|
||||
* [Database Structure](architecure/database-structure/README.md)
|
||||
* [Type of Song](architecure/database-structure/type-of-song.md)
|
||||
* [Message Queue](architecure/message-queue.md)
|
||||
* [Artificial Intelligence](architecure/artificial-intelligence.md)
|
||||
|
||||
## API Doc
|
||||
|
||||
* [Catalog](api-doc/catalog.md)
|
||||
* [Songs](api-doc/songs.md)
|
||||
- [Catalog](api-doc/catalog.md)
|
||||
- [Songs](api-doc/songs.md)
|
||||
|
||||
@ -7,34 +7,23 @@ For a **song**, it must meet the following conditions to be included in CVSA:
|
||||
|
||||
### Category 30
|
||||
|
||||
In principle, the songs must be featured in a video that is categorized under the VOCALOID·UTAU (ID 30) category in
|
||||
[Bilibili](https://en.wikipedia.org/wiki/Bilibili) in order to be observed by our
|
||||
[automation program](../architecure/overview.md#crawler). We welcome editors to manually add songs that have not been
|
||||
uploaded to bilibili / categorized under this category.
|
||||
In principle, the songs must be featured in a video that is categorized under the VOCALOID·UTAU (ID 30) category in [Bilibili](https://en.wikipedia.org/wiki/Bilibili) in order to be observed by our [automation program](../architecure/overview.md#crawler). We welcome editors to manually add songs that have not been uploaded to bilibili / categorized under this category.
|
||||
|
||||
#### NEWS
|
||||
|
||||
Recently, Bilibili seems to be offlining the sub-category. This means the VOCALOID·UTAU category can no longer be
|
||||
entered from the frontend, and producers can no longer upload videos to this category (instead, they can only choose the
|
||||
parent category "Music"). 
|
||||
Recently, Bilibili seems to be offlining the sub-category. This means the VOCALOID·UTAU category can no longer be entered from the frontend, and producers can no longer upload videos to this category (instead, they can only choose the parent category "Music"). 
|
||||
|
||||
According to our experiments, Bilibili still retains the code logic of sub-categories in the backend, and newly
|
||||
published songs may still be in the VOCALOID·UTAU sub-category, and the related APIs can still work normally. However,
|
||||
there are [reports](https://www.bilibili.com/opus/1041223385394184199) that some of the new songs have been placed under
|
||||
the "Music General" sub-category.\
|
||||
We are still waiting for Bilibili's follow-up actions, and in the future, we may adjust the scope of our automated
|
||||
program's crawling.
|
||||
According to our experiments, Bilibili still retains the code logic of sub-categories in the backend, and newly published songs may still be in the VOCALOID·UTAU sub-category, and the related APIs can still work normally. However, there are [reports](https://www.bilibili.com/opus/1041223385394184199) that some of the new songs have been placed under the "Music General" sub-category.\
|
||||
We are still waiting for Bilibili's follow-up actions, and in the future, we may adjust the scope of our automated program's crawling.
|
||||
|
||||
### At Leats One Line of Chinese / Chinese Virtual Singer
|
||||
|
||||
The lyrics of the song must contain at least one line in Chinese. Otherwise, if the lyrics of the song do not contain
|
||||
Chinese, it will only be included in the CVSA only if a Chinese virtual singer has been used.
|
||||
The lyrics of the song must contain at least one line in Chinese. Otherwise, if the lyrics of the song do not contain Chinese, it will only be included in the CVSA only if a Chinese virtual singer has been used.
|
||||
|
||||
We define a **Chinese virtual singer** as follows:
|
||||
|
||||
1. The singer primarily uses Chinese voicebank (i.e. the most widely used voickbank for the singer is Chinese)
|
||||
2. The singer is operated by a company, organization, individual or group located in Mainland China, Hong Kong, Macau or
|
||||
Taiwan.
|
||||
2. The singer is operated by a company, organization, individual or group located in Mainland China, Hong Kong, Macau or Taiwan.
|
||||
|
||||
### Using Vocal Synthesizer
|
||||
|
||||
|
||||
@ -9,13 +9,10 @@ The AI systems we currently use are:
|
||||
Located at `/filter/` under project root dir, it classifies a video in the
|
||||
[category 30](../about/scope-of-inclusion.md#category-30) into the following categories:
|
||||
|
||||
- 0: Not related to Chinese vocal synthesis
|
||||
- 1: A original song with Chinese vocal synthesis
|
||||
- 2: A cover/remix song with Chinese vocal synthesis
|
||||
* 0: Not related to Chinese vocal synthesis
|
||||
* 1: A original song with Chinese vocal synthesis
|
||||
* 2: A cover/remix song with Chinese vocal synthesis
|
||||
|
||||
### The Predictor
|
||||
|
||||
Located at `/pred/`under the project root dir, it predicts the future views of a video. This is a regression model that
|
||||
takes historical view trends of a video, other contextual information (such as the current time), and future time points
|
||||
to be predicted as feature inputs, and outputs the increment in the video's view count from "now" to the specified
|
||||
future time point.
|
||||
Located at `/pred/`under the project root dir, it predicts the future views of a video. This is a regression model that takes historical view trends of a video, other contextual information (such as the current time), and future time points to be predicted as feature inputs, and outputs the increment in the video's view count from "now" to the specified future time point.
|
||||
|
||||
@ -1,4 +0,0 @@
|
||||
# Crawler
|
||||
|
||||
A central aspect of CVSA's technical design is its emphasis on automation. The data collection process within the `crawler` is orchestrated using a message queue powered by [BullMQ](https://bullmq.io/). This enables concurrent processing of various tasks involved in the data lifecycle. State management and data persistence are handled by a combination of Redis for caching and real-time data, and PostgreSQL as the primary database.
|
||||
|
||||
@ -5,11 +5,10 @@ CVSA uses [PostgreSQL](https://www.postgresql.org/) as our database.
|
||||
All public data of CVSA (excluding users' personal data) is stored in a database named `cvsa_main`, which contains the
|
||||
following tables:
|
||||
|
||||
- songs: stores the main information of songs
|
||||
- bili\_user: stores snapshots of Bilibili user information
|
||||
- all\_data: metadata of all videos in [category 30](../../about/scope-of-inclusion.md#category-30).
|
||||
- labelling\_result: Contains label of videos in `all_data`tagged by our
|
||||
[AI system](../artificial-intelligence.md#the-filter).
|
||||
- video\_snapshot: Statistical data of videos that are fetched regularly (e.g., number of views, etc.), we call this
|
||||
fetch process as "snapshot".
|
||||
- snapshot\_schedule: The scheduling information for video snapshots.
|
||||
* songs: stores the main information of songs
|
||||
* bili\_user: stores snapshots of Bilibili user information
|
||||
* all\_data: metadata of all videos in [category 30](../../about/scope-of-inclusion.md#category-30).
|
||||
* labelling\_result: Contains label of videos in `all_data`tagged by our [AI system](../artificial-intelligence.md#the-filter).
|
||||
* video\_snapshot: Statistical data of videos that are fetched regularly (e.g., number of views, etc.), we call this fetch process as "snapshot".
|
||||
* snapshot\_schedule: The scheduling information for video snapshots.
|
||||
|
||||
|
||||
7
doc/en/architecure/message-queue.md
Normal file
7
doc/en/architecure/message-queue.md
Normal file
@ -0,0 +1,7 @@
|
||||
# Message Queue
|
||||
|
||||
We rely on message queues to manage the various tasks that [the cralwer ](overview.md#crawler)needs to perform.
|
||||
|
||||
### Code Path
|
||||
|
||||
Currently, the code related to message queues are located at `lib/mq` and `src`.
|
||||
@ -14,29 +14,14 @@ layout:
|
||||
|
||||
# Overview
|
||||
|
||||
The CVSA is a [monorepo](https://en.wikipedia.org/wiki/Monorepo) codebase, mainly using TypeScript as the development language. With [Deno workspace](https://docs.deno.com/runtime/fundamentals/workspaces/), the major part of the codebase is under `packages/`. 
|
||||
The whole CVSA system can be sperate into three different parts:
|
||||
|
||||
**Project structure:**
|
||||
* Frontend
|
||||
* API
|
||||
* Crawler
|
||||
|
||||
```
|
||||
cvsa
|
||||
├── deno.json
|
||||
├── packages
|
||||
│ ├── backend
|
||||
│ ├── core
|
||||
│ ├── crawler
|
||||
│ └── frontend
|
||||
└── README.md
|
||||
```
|
||||
|
||||
**Package Breakdown:**
|
||||
|
||||
* **`backend`**: This package houses the server-side logic, built with the [Hono](https://hono.dev/) web framework. It's responsible for interacting with the database and exposing data through REST and GraphQL APIs for consumption by the frontend, internal applications, and third-party developers.
|
||||
* **`frontend`**: The user-facing web interface of CVSA is developed using [Astro](https://astro.build/). This package handles the presentation layer, displaying information fetched from the database.
|
||||
* **`crawler`**: This automated data collection system is a key component of CVSA. It's designed to automatically discover and gather new song data from bilibili, as well as track relevant statistics over time.
|
||||
* **`core`**: This package contains reusable and generic code that is utilized across multiple workspaces within the CVSA monorepo.
|
||||
The frontend is driven by [Astro](https://astro.build/) and is used to display the final CVSA page. The API is driven by [Hono](https://hono.dev) and is used to query the database and provide REST/GraphQL APIs that can be called by out website, applications, or third parties. The crawler is our automatic data collector, used to automatically collect new songs from bilibili, track their statistics, etc.
|
||||
|
||||
### Crawler
|
||||
|
||||
Automation is the biggest highlight of CVSA's technical design. The data collection process within the `crawler` is orchestrated using a message queue powered by [BullMQ](https://bullmq.io/). This enables concurrent processing of various tasks involved in the data collection lifecycle. State management and data persistence are handled by a combination of Redis for caching and real-time data, and PostgreSQL as the primary database.
|
||||
|
||||
Automation is the biggest highlight of CVSA's technical design. To achieve this, we use a message queue powered by [BullMQ](https://bullmq.io/) to concurrently process various tasks in the data collection life cycle.
|
||||
|
||||
@ -1,106 +0,0 @@
|
||||
openapi: 3.0.0
|
||||
info:
|
||||
title: CVSA API
|
||||
version: v1
|
||||
|
||||
servers:
|
||||
- url: https://api.projectcvsa.com
|
||||
|
||||
paths:
|
||||
/video/{id}/snapshots:
|
||||
get:
|
||||
summary: 获取视频快照列表
|
||||
description: 根据视频 ID 获取视频的快照列表。视频 ID 可以是以 "av" 开头的数字，以 "BV" 开头的 12 位字母数字，或者一个正整数。
|
||||
parameters:
|
||||
- in: path
|
||||
name: id
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
description: "视频 ID (如: av78977256, BV1KJ411C7CW, 78977256)"
|
||||
- in: query
|
||||
name: ps
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: 每页返回的快照数量 (pageSize)，默认为 1000。
|
||||
- in: query
|
||||
name: pn
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: 页码 (pageNumber)，用于分页查询。offset 与 pn 只能选择一个。
|
||||
- in: query
|
||||
name: offset
|
||||
schema:
|
||||
type: integer
|
||||
minimum: 1
|
||||
description: 偏移量，用于基于偏移量的查询。offset 与 pn 只能选择一个。
|
||||
- in: query
|
||||
name: reverse
|
||||
schema:
|
||||
type: boolean
|
||||
description: 是否反向排序（从旧到新），默认为 false。
|
||||
responses:
|
||||
'200':
|
||||
description: 成功获取快照列表
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: array
|
||||
items:
|
||||
type: object
|
||||
properties:
|
||||
id:
|
||||
type: integer
|
||||
description: 快照 ID
|
||||
aid:
|
||||
type: integer
|
||||
description: 视频的 av 号
|
||||
views:
|
||||
type: integer
|
||||
description: 视频播放量
|
||||
coins:
|
||||
type: integer
|
||||
description: 视频投币数
|
||||
likes:
|
||||
type: integer
|
||||
description: 视频点赞数
|
||||
favorites:
|
||||
type: integer
|
||||
description: 视频收藏数
|
||||
shares:
|
||||
type: integer
|
||||
description: 视频分享数
|
||||
danmakus:
|
||||
type: integer
|
||||
description: 视频弹幕数
|
||||
replies:
|
||||
type: integer
|
||||
description: 视频评论数
|
||||
'400':
|
||||
description: 无效的查询参数
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
description: 错误消息
|
||||
errors:
|
||||
type: object
|
||||
description: 详细的错误信息
|
||||
'500':
|
||||
description: 服务器内部错误
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
type: object
|
||||
properties:
|
||||
message:
|
||||
type: string
|
||||
description: 错误消息
|
||||
error:
|
||||
type: object
|
||||
description: 详细的错误信息
|
||||
@ -1,22 +1,22 @@
|
||||
# Table of contents
|
||||
|
||||
* [欢迎](README.md)
|
||||
- [欢迎](README.md)
|
||||
|
||||
## 关于 <a href="#about" id="about"></a>
|
||||
|
||||
* [关于本项目](about/this-project.md)
|
||||
* [收录范围](about/scope-of-inclusion.md)
|
||||
- [关于本项目](about/this-project.md)
|
||||
- [收录范围](about/scope-of-inclusion.md)
|
||||
|
||||
## 技术架构 <a href="#architecture" id="architecture"></a>
|
||||
|
||||
- [概览](architecture/overview.md)
|
||||
- [数据库结构](architecture/database-structure/README.md)
|
||||
- [歌曲类型](architecture/database-structure/type-of-song.md)
|
||||
- [人工智能](architecture/artificial-intelligence.md)
|
||||
- [消息队列](architecture/message-queue/README.md)
|
||||
- [LatestVideosQueue 队列](architecture/message-queue/latestvideosqueue-dui-lie.md)
|
||||
* [概览](architecture/overview.md)
|
||||
* [数据库结构](architecture/database-structure/README.md)
|
||||
* [歌曲类型](architecture/database-structure/type-of-song.md)
|
||||
* [人工智能](architecture/artificial-intelligence.md)
|
||||
* [消息队列](architecture/message-queue/README.md)
|
||||
* [LatestVideosQueue 队列](architecture/message-queue/latestvideosqueue-dui-lie.md)
|
||||
|
||||
## API 文档 <a href="#api-doc" id="api-doc"></a>
|
||||
|
||||
* [目录](api-doc/catalog.md)
|
||||
* [视频快照](api-doc/video-snapshot.md)
|
||||
- [目录](api-doc/catalog.md)
|
||||
- [歌曲](api-doc/songs.md)
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
# 目录
|
||||
|
||||
* [视频快照](video-snapshot.md)
|
||||
|
||||
- [歌曲](songs.md)
|
||||
|
||||
3
doc/zh/api-doc/songs.md
Normal file
3
doc/zh/api-doc/songs.md
Normal file
@ -0,0 +1,3 @@
|
||||
# 歌曲
|
||||
|
||||
暂未实现。
|
||||
@ -1,6 +0,0 @@
|
||||
# 视频快照
|
||||
|
||||
{% openapi src="../.gitbook/assets/1.yaml" path="/video/{id}/snapshots" method="get" %}
|
||||
[1.yaml](../.gitbook/assets/1.yaml)
|
||||
{% endopenapi %}
|
||||
|
||||
@ -2,14 +2,13 @@
|
||||
|
||||
CVSA 使用 [PostgreSQL](https://www.postgresql.org/) 作为数据库。
|
||||
|
||||
CVSA 设计了两个
|
||||
|
||||
CVSA 的所有公开数据（不包括用户的个人数据）都存储在名为 `cvsa_main` 的数据库中，该数据库包含以下表：
|
||||
|
||||
- songs：存储歌曲的主要信息
|
||||
- bilibili\_user：存储 Bilibili 用户信息快照
|
||||
- bilibili\_metadata：[分区 30](../../about/scope-of-inclusion.md#vocaloiduatu-fen-qu) 中所有视频的元数据
|
||||
- labelling\_result：包含由我们的 AI 系统 标记的 `all_data` 中视频的标签。
|
||||
- latest\_video\_snapshot：存储视频最新的快照
|
||||
- video\_snapshot：存储视频的快照，包括特定时间下视频的统计信息（播放量、点赞数等）
|
||||
- snapshot\_schedule：视频快照的规划信息，为辅助表
|
||||
* songs：存储歌曲的主要信息
|
||||
* bilibili\_user：存储 Bilibili 用户信息快照
|
||||
* bilibili\_metadata：[分区 30](../../about/scope-of-inclusion.md#vocaloiduatu-fen-qu) 中所有视频的元数据
|
||||
* labelling\_result：包含由我们的 AI 系统 标记的 `all_data` 中视频的标签。
|
||||
* latest\_video\_snapshot：存储视频最新的快照
|
||||
* video\_snapshot：存储视频的快照，包括特定时间下视频的统计信息（播放量、点赞数等）
|
||||
* snapshot\_schedule：视频快照的规划信息，为辅助表
|
||||
|
||||
|
||||
@ -1 +1,2 @@
|
||||
# LatestVideosQueue 队列
|
||||
|
||||
|
||||
@ -20,7 +20,8 @@ layout:
|
||||
|
||||
位于项目目录`packages/crawler` 下，它负责以下工作：
|
||||
|
||||
- 抓取新的视频并收录作品
|
||||
- 持续监控视频的播放量等统计信息
|
||||
* 抓取新的视频并收录作品
|
||||
* 持续监控视频的播放量等统计信息
|
||||
|
||||
整个 crawler 由 BullMQ 消息队列驱动，使用 Redis 和 PostgreSQL 管理状态。
|
||||
|
||||
|
||||
@ -9,18 +9,18 @@ export const db = pool;
|
||||
export const dbCred = poolCred;
|
||||
|
||||
export const dbMiddleware = createMiddleware(async (c, next) => {
|
||||
const connection = await pool.connect();
|
||||
const connection = await pool.connect();
|
||||
c.set("db", connection);
|
||||
await next();
|
||||
connection.release();
|
||||
});
|
||||
|
||||
export const dbCredMiddleware = createMiddleware(async (c, next) => {
|
||||
const connection = await poolCred.connect();
|
||||
const connection = await poolCred.connect();
|
||||
c.set("dbCred", connection);
|
||||
await next();
|
||||
connection.release();
|
||||
});
|
||||
})
|
||||
|
||||
declare module "hono" {
|
||||
interface ContextVariableMap {
|
||||
|
||||
@ -4,15 +4,11 @@
|
||||
"@rabbit-company/argon2id": "jsr:@rabbit-company/argon2id@^2.1.0",
|
||||
"hono": "jsr:@hono/hono@^4.7.5",
|
||||
"zod": "npm:zod",
|
||||
"yup": "npm:yup",
|
||||
"@core/": "../core/",
|
||||
"log/": "../core/log/",
|
||||
"@crawler/net/videoInfo": "../crawler/net/getVideoInfo.ts",
|
||||
"ioredis": "npm:ioredis"
|
||||
"yup": "npm:yup"
|
||||
},
|
||||
"tasks": {
|
||||
"dev": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --watch main.ts",
|
||||
"start": "deno serve --env-file=.env --allow-env --allow-net --allow-read --allow-write --allow-run --host 127.0.0.1 main.ts"
|
||||
"dev": "deno serve --env-file=.env --allow-env --allow-net --watch main.ts",
|
||||
"start": "deno serve --env-file=.env --allow-env --allow-net --host 127.0.0.1 main.ts"
|
||||
},
|
||||
"compilerOptions": {
|
||||
"jsx": "precompile",
|
||||
|
||||
@ -3,19 +3,16 @@ import { dbCredMiddleware, dbMiddleware } from "./database.ts";
|
||||
import { rootHandler } from "./root.ts";
|
||||
import { getSnapshotsHanlder } from "./snapshots.ts";
|
||||
import { registerHandler } from "./register.ts";
|
||||
import { videoInfoHandler } from "./videoInfo.ts";
|
||||
|
||||
export const app = new Hono();
|
||||
|
||||
app.use("/video/*", dbMiddleware);
|
||||
app.use("/user", dbCredMiddleware);
|
||||
app.use('/video/*', dbMiddleware);
|
||||
app.use('/user', dbCredMiddleware);
|
||||
|
||||
app.get("/", ...rootHandler);
|
||||
|
||||
app.get("/video/:id/snapshots", ...getSnapshotsHanlder);
|
||||
app.post("/user", ...registerHandler);
|
||||
|
||||
app.get("/video/:id/info", ...videoInfoHandler);
|
||||
app.get('/video/:id/snapshots', ...getSnapshotsHanlder);
|
||||
app.post('/user', ...registerHandler);
|
||||
|
||||
const fetch = app.fetch;
|
||||
|
||||
@ -23,4 +20,4 @@ export default {
|
||||
fetch,
|
||||
} satisfies Deno.ServeDefaultExport;
|
||||
|
||||
export const VERSION = "0.4.2";
|
||||
export const VERSION = "0.3.0";
|
||||
@ -8,7 +8,7 @@ import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
const RegistrationBodySchema = object({
|
||||
username: string().trim().required("Username is required").max(50, "Username cannot exceed 50 characters"),
|
||||
password: string().required("Password is required"),
|
||||
nickname: string().optional(),
|
||||
nickname: string().optional(),
|
||||
});
|
||||
|
||||
type ContextType = Context<BlankEnv & { Bindings: Bindings }, "/user", BlankInput>;
|
||||
@ -19,7 +19,7 @@ export const userExists = async (username: string, client: Client) => {
|
||||
`;
|
||||
const result = await client.queryObject(query, [username]);
|
||||
return result.rows.length > 0;
|
||||
};
|
||||
}
|
||||
|
||||
export const registerHandler = createHandlers(async (c: ContextType) => {
|
||||
const client = c.get("dbCred");
|
||||
@ -28,11 +28,11 @@ export const registerHandler = createHandlers(async (c: ContextType) => {
|
||||
const body = await RegistrationBodySchema.validate(await c.req.json());
|
||||
const { username, password, nickname } = body;
|
||||
|
||||
if (await userExists(username, client)) {
|
||||
if (await userExists(username, client)) {
|
||||
return c.json({
|
||||
message: `User "${username}" already exists.`,
|
||||
}, 400);
|
||||
}
|
||||
}
|
||||
|
||||
const hash = await Argon2id.hashEncoded(password);
|
||||
|
||||
@ -49,7 +49,7 @@ export const registerHandler = createHandlers(async (c: ContextType) => {
|
||||
return c.json({
|
||||
message: "Invalid registration data.",
|
||||
errors: e.errors,
|
||||
}, 400);
|
||||
}, 400);
|
||||
} else if (e instanceof SyntaxError) {
|
||||
return c.json({
|
||||
message: "Invalid JSON in request body.",
|
||||
|
||||
@ -3,27 +3,29 @@ import { VERSION } from "./main.ts";
|
||||
import { createHandlers } from "./utils.ts";
|
||||
|
||||
export const rootHandler = createHandlers((c) => {
|
||||
let singer: Singer | Singer[] | null;
|
||||
let singer: Singer | Singer[] | null = null;
|
||||
const shouldShowSpecialSinger = Math.random() < 0.016;
|
||||
if (getSingerForBirthday().length !== 0) {
|
||||
if (getSingerForBirthday().length !== 0){
|
||||
singer = getSingerForBirthday();
|
||||
for (const s of singer) {
|
||||
delete s.birthday;
|
||||
s.message = `祝${s.name}生日快乐~`;
|
||||
s.message = `祝${s.name}生日快乐~`
|
||||
}
|
||||
} else if (shouldShowSpecialSinger) {
|
||||
singer = pickSpecialSinger();
|
||||
} else {
|
||||
singer = pickSinger();
|
||||
}
|
||||
else if (shouldShowSpecialSinger) {
|
||||
singer = pickSpecialSinger();
|
||||
}
|
||||
else {
|
||||
singer = pickSinger();
|
||||
}
|
||||
return c.json({
|
||||
"project": {
|
||||
"name": "中V档案馆",
|
||||
"motto": "一起唱吧，心中的歌！",
|
||||
"motto": "一起唱吧，心中的歌！"
|
||||
},
|
||||
"status": 200,
|
||||
"version": VERSION,
|
||||
"time": Date.now(),
|
||||
"singer": singer,
|
||||
});
|
||||
});
|
||||
"singer": singer
|
||||
})
|
||||
})
|
||||
@ -70,7 +70,7 @@ export interface Singer {
|
||||
name: string;
|
||||
color?: string;
|
||||
birthday?: string;
|
||||
message?: string;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export const specialSingers = [
|
||||
|
||||
@ -12,12 +12,12 @@ const SnapshotQueryParamsSchema = object({
|
||||
reverse: boolean().optional(),
|
||||
});
|
||||
|
||||
export const idSchema = mixed().test(
|
||||
const idSchema = mixed().test(
|
||||
"is-valid-id",
|
||||
'id must be a string starting with "av" followed by digits, or "BV" followed by 10 alphanumeric characters, or a positive integer',
|
||||
async (value) => {
|
||||
if (value && await number().integer().isValid(value)) {
|
||||
const v = parseInt(value as string);
|
||||
const v = parseInt(value as string);
|
||||
return Number.isInteger(v) && v > 0;
|
||||
}
|
||||
|
||||
@ -46,9 +46,10 @@ export const getSnapshotsHanlder = createHandlers(async (c: ContextType) => {
|
||||
let videoId: string | number = idParam as string;
|
||||
if (videoId.startsWith("av")) {
|
||||
videoId = parseInt(videoId.slice(2));
|
||||
} else if (await number().isValid(videoId)) {
|
||||
videoId = parseInt(videoId);
|
||||
}
|
||||
else if (await number().isValid(videoId)) {
|
||||
videoId = parseInt(videoId);
|
||||
}
|
||||
const queryParams = await SnapshotQueryParamsSchema.validate(c.req.query());
|
||||
const { ps, pn, offset, reverse = false } = queryParams;
|
||||
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { createFactory } from "hono/factory";
|
||||
import { createFactory } from 'hono/factory'
|
||||
|
||||
const factory = createFactory();
|
||||
|
||||
export const createHandlers = factory.createHandlers;
|
||||
export const createHandlers = factory.createHandlers;
|
||||
@ -1,86 +0,0 @@
|
||||
import logger from "log/logger.ts";
|
||||
import { Redis } from "ioredis";
|
||||
import { number, ValidationError } from "yup";
|
||||
import { createHandlers } from "./utils.ts";
|
||||
import { getVideoInfo, getVideoInfoByBV } from "@crawler/net/videoInfo";
|
||||
import { idSchema } from "./snapshots.ts";
|
||||
import { NetSchedulerError } from "@core/net/delegate.ts";
|
||||
import type { Context } from "hono";
|
||||
import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import type { BlankEnv, BlankInput } from "hono/types";
|
||||
import type { VideoInfoData } from "@core/net/bilibili.d.ts";
|
||||
|
||||
|
||||
const redis = new Redis({ maxRetriesPerRequest: null });
|
||||
const CACHE_EXPIRATION_SECONDS = 60;
|
||||
|
||||
type ContextType = Context<BlankEnv, "/video/:id/info", BlankInput>;
|
||||
|
||||
async function insertVideoSnapshot(client: Client, data: VideoInfoData) {
|
||||
const views = data.stat.view;
|
||||
const danmakus = data.stat.danmaku;
|
||||
const replies = data.stat.reply;
|
||||
const likes = data.stat.like;
|
||||
const coins = data.stat.coin;
|
||||
const shares = data.stat.share;
|
||||
const favorites = data.stat.favorite;
|
||||
const aid = data.aid;
|
||||
|
||||
const query: string = `
|
||||
INSERT INTO video_snapshot (aid, views, danmakus, replies, likes, coins, shares, favorites)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
`;
|
||||
|
||||
await client.queryObject(
|
||||
query,
|
||||
[aid, views, danmakus, replies, likes, coins, shares, favorites],
|
||||
);
|
||||
|
||||
logger.log(`Inserted into snapshot for video ${aid} by videoInfo API.`, "api", "fn:insertVideoSnapshot");
|
||||
}
|
||||
|
||||
|
||||
export const videoInfoHandler = createHandlers(async (c: ContextType) => {
|
||||
const client = c.get("db");
|
||||
try {
|
||||
const id = await idSchema.validate(c.req.param("id"));
|
||||
let videoId: string | number = id as string;
|
||||
if (videoId.startsWith("av")) {
|
||||
videoId = parseInt(videoId.slice(2));
|
||||
} else if (await number().isValid(videoId)) {
|
||||
videoId = parseInt(videoId);
|
||||
}
|
||||
|
||||
const cacheKey = `cvsa:videoInfo:${videoId}`;
|
||||
const cachedData = await redis.get(cacheKey);
|
||||
|
||||
if (cachedData) {
|
||||
return c.json(JSON.parse(cachedData));
|
||||
}
|
||||
|
||||
let result: VideoInfoData | number;
|
||||
if (typeof videoId === "number") {
|
||||
result = await getVideoInfo(videoId, "getVideoInfo");
|
||||
} else {
|
||||
result = await getVideoInfoByBV(videoId, "getVideoInfo");
|
||||
}
|
||||
|
||||
if (typeof result === "number") {
|
||||
return c.json({ message: "Error fetching video info", code: result }, 500);
|
||||
}
|
||||
|
||||
await redis.setex(cacheKey, CACHE_EXPIRATION_SECONDS, JSON.stringify(result));
|
||||
|
||||
await insertVideoSnapshot(client, result);
|
||||
|
||||
return c.json(result);
|
||||
} catch (e) {
|
||||
if (e instanceof ValidationError) {
|
||||
return c.json({ message: "Invalid query parameters", errors: e.errors }, 400);
|
||||
} else if (e instanceof NetSchedulerError) {
|
||||
return c.json({ message: "Error fetching video info", code: e.code }, 500);
|
||||
} else {
|
||||
return c.json({ message: "Unhandled error", error: e }, 500);
|
||||
}
|
||||
}
|
||||
});
|
||||
@ -1,62 +1,33 @@
|
||||
import type { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import type { VideoSnapshotType } from "./schema.d.ts";
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { VideoSnapshotType } from "@core/db/schema.d.ts";
|
||||
|
||||
export async function getVideoSnapshots(
|
||||
client: Client,
|
||||
aid: number,
|
||||
limit: number,
|
||||
pageOrOffset: number,
|
||||
reverse: boolean,
|
||||
mode: "page" | "offset" = "page",
|
||||
) {
|
||||
const offset = mode === "page" ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||
const queryDesc: string = `
|
||||
export async function getVideoSnapshots(client: Client, aid: number, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
|
||||
const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||
const order = reverse ? 'ASC' : 'DESC';
|
||||
const query = `
|
||||
SELECT *
|
||||
FROM video_snapshot
|
||||
WHERE aid = $1
|
||||
ORDER BY created_at DESC
|
||||
ORDER BY created_at ${order}
|
||||
LIMIT $2
|
||||
OFFSET $3
|
||||
`;
|
||||
const queryAsc: string = `
|
||||
SELECT *
|
||||
FROM video_snapshot
|
||||
WHERE aid = $1
|
||||
ORDER BY created_at
|
||||
LIMIT $2 OFFSET $3
|
||||
`;
|
||||
const query = reverse ? queryAsc : queryDesc;
|
||||
const queryResult = await client.queryObject<VideoSnapshotType>(query, [aid, limit, offset]);
|
||||
return queryResult.rows;
|
||||
const queryResult = await client.queryObject<VideoSnapshotType>(query, [aid, limit, offset]);
|
||||
return queryResult.rows;
|
||||
}
|
||||
|
||||
export async function getVideoSnapshotsByBV(
|
||||
client: Client,
|
||||
bv: string,
|
||||
limit: number,
|
||||
pageOrOffset: number,
|
||||
reverse: boolean,
|
||||
mode: "page" | "offset" = "page",
|
||||
) {
|
||||
const offset = mode === "page" ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||
const queryAsc = `
|
||||
export async function getVideoSnapshotsByBV(client: Client, bv: string, limit: number, pageOrOffset: number, reverse: boolean, mode: 'page' | 'offset' = 'page') {
|
||||
const offset = mode === 'page' ? (pageOrOffset - 1) * limit : pageOrOffset;
|
||||
const order = reverse ? 'ASC' : 'DESC';
|
||||
const query = `
|
||||
SELECT vs.*
|
||||
FROM video_snapshot vs
|
||||
JOIN bilibili_metadata bm ON vs.aid = bm.aid
|
||||
WHERE bm.bvid = $1
|
||||
ORDER BY vs.created_at
|
||||
ORDER BY vs.created_at ${order}
|
||||
LIMIT $2
|
||||
OFFSET $3
|
||||
`;
|
||||
const queryDesc: string = `
|
||||
SELECT *
|
||||
FROM video_snapshot vs
|
||||
JOIN bilibili_metadata bm ON vs.aid = bm.aid
|
||||
WHERE bm.bvid = $1
|
||||
ORDER BY vs.created_at DESC
|
||||
LIMIT $2 OFFSET $3
|
||||
`;
|
||||
const query = reverse ? queryAsc : queryDesc;
|
||||
const queryResult = await client.queryObject<VideoSnapshotType>(query, [bv, limit, offset]);
|
||||
return queryResult.rows;
|
||||
}
|
||||
`
|
||||
const queryResult = await client.queryObject<VideoSnapshotType>(query, [bv, limit, offset]);
|
||||
return queryResult.rows;
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
{
|
||||
"name": "@cvsa/core",
|
||||
"exports": "./main.ts",
|
||||
"imports": {
|
||||
"ioredis": "npm:ioredis",
|
||||
"log/": "./log/",
|
||||
"db/": "./db/",
|
||||
"$std/": "https://deno.land/std@0.216.0/",
|
||||
"mq/": "./mq/",
|
||||
"chalk": "npm:chalk"
|
||||
}
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
export const DB_VERSION = 10;
|
||||
@ -1,5 +1,5 @@
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { AllDataType, BiliUserType } from "@core/db/schema";
|
||||
import { AllDataType, BiliUserType } from "db/schema.d.ts";
|
||||
import Akari from "ml/akari.ts";
|
||||
|
||||
export async function videoExistsInAllData(client: Client, aid: number) {
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Pool } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { postgresConfig } from "@core/db/pgConfig";
|
||||
import { postgresConfig } from "@core/db/pgConfig.ts";
|
||||
|
||||
const pool = new Pool(postgresConfig, 12);
|
||||
|
||||
|
||||
55
packages/crawler/db/schema.d.ts
vendored
Normal file
55
packages/crawler/db/schema.d.ts
vendored
Normal file
@ -0,0 +1,55 @@
|
||||
export interface AllDataType {
|
||||
id: number;
|
||||
aid: number;
|
||||
bvid: string | null;
|
||||
description: string | null;
|
||||
uid: number | null;
|
||||
tags: string | null;
|
||||
title: string | null;
|
||||
published_at: string | null;
|
||||
duration: number;
|
||||
created_at: string | null;
|
||||
}
|
||||
|
||||
export interface BiliUserType {
|
||||
id: number;
|
||||
uid: number;
|
||||
username: string;
|
||||
desc: string;
|
||||
fans: number;
|
||||
}
|
||||
|
||||
export interface VideoSnapshotType {
|
||||
id: number;
|
||||
created_at: string;
|
||||
views: number;
|
||||
coins: number;
|
||||
likes: number;
|
||||
favorites: number;
|
||||
shares: number;
|
||||
danmakus: number;
|
||||
aid: bigint;
|
||||
replies: number;
|
||||
}
|
||||
|
||||
export interface LatestSnapshotType {
|
||||
aid: number;
|
||||
time: number;
|
||||
views: number;
|
||||
danmakus: number;
|
||||
replies: number;
|
||||
likes: number;
|
||||
coins: number;
|
||||
shares: number;
|
||||
favorites: number;
|
||||
}
|
||||
|
||||
export interface SnapshotScheduleType {
|
||||
id: number;
|
||||
aid: number;
|
||||
type?: string;
|
||||
created_at: string;
|
||||
started_at?: string;
|
||||
finished_at?: string;
|
||||
status: string;
|
||||
}
|
||||
@ -1,13 +1,12 @@
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { LatestSnapshotType } from "@core/db/schema";
|
||||
import { SnapshotNumber } from "mq/task/getVideoStats.ts";
|
||||
import { LatestSnapshotType } from "db/schema.d.ts";
|
||||
|
||||
export async function getVideosNearMilestone(client: Client) {
|
||||
const queryResult = await client.queryObject<LatestSnapshotType>(`
|
||||
SELECT ls.*
|
||||
FROM latest_video_snapshot ls
|
||||
WHERE
|
||||
views < 100000 OR
|
||||
(views >= 90000 AND views < 100000) OR
|
||||
(views >= 900000 AND views < 1000000) OR
|
||||
(views >= 9900000 AND views < 10000000)
|
||||
`);
|
||||
@ -19,7 +18,7 @@ export async function getVideosNearMilestone(client: Client) {
|
||||
});
|
||||
}
|
||||
|
||||
export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | SnapshotNumber> {
|
||||
export async function getLatestVideoSnapshot(client: Client, aid: number): Promise<null | LatestSnapshotType> {
|
||||
const queryResult = await client.queryObject<LatestSnapshotType>(
|
||||
`
|
||||
SELECT *
|
||||
|
||||
@ -1,8 +1,9 @@
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { SnapshotScheduleType } from "@core/db/schema";
|
||||
import { formatTimestampToPsql } from "utils/formatTimestampToPostgre.ts";
|
||||
import { SnapshotScheduleType } from "./schema.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
import { MINUTE } from "$std/datetime/constants.ts";
|
||||
import { redis } from "../../core/db/redis.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
import { Redis } from "ioredis";
|
||||
|
||||
const REDIS_KEY = "cvsa:snapshot_window_counts";
|
||||
@ -10,7 +11,8 @@ const REDIS_KEY = "cvsa:snapshot_window_counts";
|
||||
function getCurrentWindowIndex(): number {
|
||||
const now = new Date();
|
||||
const minutesSinceMidnight = now.getHours() * 60 + now.getMinutes();
|
||||
return Math.floor(minutesSinceMidnight / 5);
|
||||
const currentWindow = Math.floor(minutesSinceMidnight / 5);
|
||||
return currentWindow;
|
||||
}
|
||||
|
||||
export async function refreshSnapshotWindowCounts(client: Client, redisClient: Redis) {
|
||||
@ -168,6 +170,24 @@ export async function getLatestSnapshot(client: Client, aid: number): Promise<Sn
|
||||
};
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns the number of snapshot schedules within the specified range.
|
||||
* @param client The database client.
|
||||
* @param start The start time of the range. (Timestamp in milliseconds)
|
||||
* @param end The end time of the range. (Timestamp in milliseconds)
|
||||
*/
|
||||
export async function getSnapshotScheduleCountWithinRange(client: Client, start: number, end: number) {
|
||||
const startTimeString = formatTimestampToPsql(start);
|
||||
const endTimeString = formatTimestampToPsql(end);
|
||||
const query = `
|
||||
SELECT COUNT(*) FROM snapshot_schedule
|
||||
WHERE started_at BETWEEN $1 AND $2
|
||||
AND status = 'pending'
|
||||
`;
|
||||
const res = await client.queryObject<{ count: number }>(query, [startTimeString, endTimeString]);
|
||||
return res.rows[0].count;
|
||||
}
|
||||
|
||||
/*
|
||||
* Creates a new snapshot schedule record.
|
||||
* @param client The database client.
|
||||
@ -215,7 +235,7 @@ export async function adjustSnapshotTime(
|
||||
|
||||
const initialOffset = currentWindow + Math.max(targetOffset, 0);
|
||||
|
||||
let timePerIteration: number;
|
||||
let timePerIteration = 0;
|
||||
const MAX_ITERATIONS = 2880;
|
||||
let iters = 0;
|
||||
const t = performance.now();
|
||||
|
||||
@ -1,32 +0,0 @@
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { db } from "db/init.ts";
|
||||
|
||||
/**
|
||||
* Executes a function with a database connection.
|
||||
* @param operation The function that accepts the `client` as the parameter.
|
||||
* @param errorHandling Optional function to handle errors.
|
||||
* If no error handling function is provided, the error will be re-thrown.
|
||||
* @param cleanup Optional function to execute after the operation.
|
||||
* @returns The result of the operation or undefined if an error occurred.
|
||||
*/
|
||||
export async function withDbConnection<T>(
|
||||
operation: (client: Client) => Promise<T>,
|
||||
errorHandling?: (error: unknown) => void,
|
||||
cleanup?: () => void,
|
||||
): Promise<T | undefined> {
|
||||
const client = await db.connect();
|
||||
try {
|
||||
return await operation(client);
|
||||
} catch (error) {
|
||||
if (errorHandling) {
|
||||
errorHandling(error);
|
||||
return;
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
client.release();
|
||||
if (cleanup) {
|
||||
cleanup();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -27,8 +27,7 @@
|
||||
"bullmq": "npm:bullmq",
|
||||
"mq/": "./mq/",
|
||||
"db/": "./db/",
|
||||
"@core/": "../core/",
|
||||
"log/": "../core/log/",
|
||||
"log/": "./log/",
|
||||
"net/": "./net/",
|
||||
"ml/": "./ml/",
|
||||
"utils/": "./utils/",
|
||||
@ -38,9 +37,7 @@
|
||||
"express": "npm:express",
|
||||
"src/": "./src/",
|
||||
"onnxruntime": "npm:onnxruntime-node@1.19.2",
|
||||
"chalk": "npm:chalk",
|
||||
"@core/db/schema": "../core/db/schema.d.ts",
|
||||
"@core/db/pgConfig": "../core/db/pgConfig.ts"
|
||||
"chalk": "npm:chalk"
|
||||
},
|
||||
"exports": "./main.ts"
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import winston, { format, transports } from "npm:winston";
|
||||
import type { TransformableInfo } from "npm:logform";
|
||||
import { TransformableInfo } from "npm:logform";
|
||||
import chalk from "chalk";
|
||||
|
||||
const customFormat = format.printf((info: TransformableInfo) => {
|
||||
@ -11,6 +11,7 @@ import {
|
||||
getLatestSnapshot,
|
||||
getSnapshotsInNextSecond,
|
||||
getVideosWithoutActiveSnapshotSchedule,
|
||||
hasAtLeast2Snapshots,
|
||||
scheduleSnapshot,
|
||||
setSnapshotStatus,
|
||||
snapshotScheduleExists,
|
||||
@ -21,13 +22,12 @@ import { HOUR, MINUTE, SECOND, WEEK } from "$std/datetime/constants.ts";
|
||||
import logger from "log/logger.ts";
|
||||
import { SnapshotQueue } from "mq/index.ts";
|
||||
import { insertVideoSnapshot } from "mq/task/getVideoStats.ts";
|
||||
import { NetSchedulerError } from "@core/net/delegate.ts";
|
||||
import { NetSchedulerError } from "mq/scheduler.ts";
|
||||
import { getBiliVideoStatus, setBiliVideoStatus } from "db/allData.ts";
|
||||
import { truncate } from "utils/truncate.ts";
|
||||
import { lockManager } from "mq/lockManager.ts";
|
||||
import { getSongsPublihsedAt } from "db/songs.ts";
|
||||
import { bulkGetVideoStats } from "net/bulkGetVideoStats.ts";
|
||||
import { getAdjustedShortTermETA } from "../scheduling.ts";
|
||||
|
||||
const priorityMap: { [key: string]: number } = {
|
||||
"milestone": 1,
|
||||
@ -103,6 +103,52 @@ export const closetMilestone = (views: number) => {
|
||||
return 10000000;
|
||||
};
|
||||
|
||||
const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
|
||||
|
||||
/*
|
||||
* Returns the minimum ETA in hours for the next snapshot
|
||||
* @param client - Postgres client
|
||||
* @param aid - aid of the video
|
||||
* @returns ETA in hours
|
||||
*/
|
||||
export const getAdjustedShortTermETA = async (client: Client, aid: number) => {
|
||||
const latestSnapshot = await getLatestSnapshot(client, aid);
|
||||
// Immediately dispatch a snapshot if there is no snapshot yet
|
||||
if (!latestSnapshot) return 0;
|
||||
const snapshotsEnough = await hasAtLeast2Snapshots(client, aid);
|
||||
if (!snapshotsEnough) return 0;
|
||||
|
||||
const currentTimestamp = new Date().getTime();
|
||||
const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
|
||||
const DELTA = 0.00001;
|
||||
let minETAHours = Infinity;
|
||||
|
||||
for (const timeInterval of timeIntervals) {
|
||||
const date = new Date(currentTimestamp - timeInterval);
|
||||
const snapshot = await findClosestSnapshot(client, aid, date);
|
||||
if (!snapshot) continue;
|
||||
const hoursDiff = (latestSnapshot.created_at - snapshot.created_at) / HOUR;
|
||||
const viewsDiff = latestSnapshot.views - snapshot.views;
|
||||
if (viewsDiff <= 0) continue;
|
||||
const speed = viewsDiff / (hoursDiff + DELTA);
|
||||
const target = closetMilestone(latestSnapshot.views);
|
||||
const viewsToIncrease = target - latestSnapshot.views;
|
||||
const eta = viewsToIncrease / (speed + DELTA);
|
||||
let factor = log(2.97 / log(viewsToIncrease + 1), 1.14);
|
||||
factor = truncate(factor, 3, 100);
|
||||
const adjustedETA = eta / factor;
|
||||
if (adjustedETA < minETAHours) {
|
||||
minETAHours = adjustedETA;
|
||||
}
|
||||
}
|
||||
|
||||
if (isNaN(minETAHours)) {
|
||||
minETAHours = Infinity;
|
||||
}
|
||||
|
||||
return minETAHours;
|
||||
};
|
||||
|
||||
export const collectMilestoneSnapshotsWorker = async (_job: Job) => {
|
||||
const client = await db.connect();
|
||||
try {
|
||||
@ -110,7 +156,7 @@ export const collectMilestoneSnapshotsWorker = async (_job: Job) => {
|
||||
for (const video of videos) {
|
||||
const aid = Number(video.aid);
|
||||
const eta = await getAdjustedShortTermETA(client, aid);
|
||||
if (eta > 144) continue;
|
||||
if (eta > 72) continue;
|
||||
const now = Date.now();
|
||||
const scheduledNextSnapshotDelay = eta * HOUR;
|
||||
const maxInterval = 4 * HOUR;
|
||||
@ -173,7 +219,7 @@ export const regularSnapshotsWorker = async (_job: Job) => {
|
||||
} catch (e) {
|
||||
logger.error(e as Error, "mq", "fn:regularSnapshotsWorker");
|
||||
} finally {
|
||||
await lockManager.releaseLock("dispatchRegularSnapshots");
|
||||
lockManager.releaseLock("dispatchRegularSnapshots");
|
||||
client.release();
|
||||
}
|
||||
};
|
||||
@ -299,7 +345,7 @@ export const takeSnapshotForVideoWorker = async (job: Job) => {
|
||||
}
|
||||
if (type !== "milestone") return `DONE`;
|
||||
const eta = await getAdjustedShortTermETA(client, aid);
|
||||
if (eta > 144) return "ETA_TOO_LONG";
|
||||
if (eta > 72) return "ETA_TOO_LONG";
|
||||
const now = Date.now();
|
||||
const targetTime = now + eta * HOUR;
|
||||
await scheduleSnapshot(client, aid, type, targetTime);
|
||||
|
||||
@ -3,7 +3,7 @@ import { ClassifyVideoQueue, LatestVideosQueue, SnapshotQueue } from "mq/index.t
|
||||
import logger from "log/logger.ts";
|
||||
import { initSnapshotWindowCounts } from "db/snapshotSchedule.ts";
|
||||
import { db } from "db/init.ts";
|
||||
import { redis } from "@core/db/redis.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
|
||||
export async function initMQ() {
|
||||
const client = await db.connect();
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { Redis } from "ioredis";
|
||||
import { redis } from "../../core/db/redis.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
|
||||
class LockManager {
|
||||
private redis: Redis;
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { SlidingWindow } from "./slidingWindow.ts";
|
||||
import { SlidingWindow } from "mq/slidingWindow.ts";
|
||||
|
||||
export interface RateLimiterConfig {
|
||||
window: SlidingWindow;
|
||||
@ -1,5 +1,5 @@
|
||||
import logger from "log/logger.ts";
|
||||
import { RateLimiter, type RateLimiterConfig } from "mq/rateLimiter.ts";
|
||||
import { RateLimiter, RateLimiterConfig } from "mq/rateLimiter.ts";
|
||||
import { SlidingWindow } from "mq/slidingWindow.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
import Redis from "ioredis";
|
||||
@ -19,7 +19,7 @@ interface ProxiesMap {
|
||||
[name: string]: Proxy;
|
||||
}
|
||||
|
||||
type NetworkDelegateErrorCode =
|
||||
type NetSchedulerErrorCode =
|
||||
| "NO_PROXY_AVAILABLE"
|
||||
| "PROXY_RATE_LIMITED"
|
||||
| "PROXY_NOT_FOUND"
|
||||
@ -28,9 +28,9 @@ type NetworkDelegateErrorCode =
|
||||
| "ALICLOUD_PROXY_ERR";
|
||||
|
||||
export class NetSchedulerError extends Error {
|
||||
public code: NetworkDelegateErrorCode;
|
||||
public code: NetSchedulerErrorCode;
|
||||
public rawError: unknown | undefined;
|
||||
constructor(message: string, errorCode: NetworkDelegateErrorCode, rawError?: unknown) {
|
||||
constructor(message: string, errorCode: NetSchedulerErrorCode, rawError?: unknown) {
|
||||
super(message);
|
||||
this.name = "NetSchedulerError";
|
||||
this.code = errorCode;
|
||||
@ -59,7 +59,7 @@ function shuffleArray<T>(array: T[]): T[] {
|
||||
return newArray;
|
||||
}
|
||||
|
||||
class NetworkDelegate {
|
||||
class NetScheduler {
|
||||
private proxies: ProxiesMap = {};
|
||||
private providerLimiters: LimiterMap = {};
|
||||
private proxyLimiters: OptionalLimiterMap = {};
|
||||
@ -69,6 +69,23 @@ class NetworkDelegate {
|
||||
this.proxies[proxyName] = { type, data };
|
||||
}
|
||||
|
||||
removeProxy(proxyName: string): void {
|
||||
if (!this.proxies[proxyName]) {
|
||||
throw new Error(`Proxy ${proxyName} not found`);
|
||||
}
|
||||
delete this.proxies[proxyName];
|
||||
// Clean up associated limiters
|
||||
this.cleanupProxyLimiters(proxyName);
|
||||
}
|
||||
|
||||
private cleanupProxyLimiters(proxyName: string): void {
|
||||
for (const limiterId in this.proxyLimiters) {
|
||||
if (limiterId.startsWith(`proxy-${proxyName}`)) {
|
||||
delete this.proxyLimiters[limiterId];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
addTask(taskName: string, provider: string, proxies: string[] | "all"): void {
|
||||
this.tasks[taskName] = { provider, proxies };
|
||||
}
|
||||
@ -200,7 +217,8 @@ class NetworkDelegate {
|
||||
const providerLimiterId = "provider-" + proxyName + "-" + provider;
|
||||
if (!this.proxyLimiters[proxyLimiterId]) {
|
||||
const providerLimiter = this.providerLimiters[providerLimiterId];
|
||||
return await providerLimiter.getAvailability();
|
||||
const providerAvailable = await providerLimiter.getAvailability();
|
||||
return providerAvailable;
|
||||
}
|
||||
const proxyLimiter = this.proxyLimiters[proxyLimiterId];
|
||||
const providerLimiter = this.providerLimiters[providerLimiterId];
|
||||
@ -263,7 +281,6 @@ class NetworkDelegate {
|
||||
const out = decoder.decode(output.stdout);
|
||||
const rawData = JSON.parse(out);
|
||||
if (rawData.statusCode !== 200) {
|
||||
// noinspection ExceptionCaughtLocallyJS
|
||||
throw new NetSchedulerError(
|
||||
`Error proxying ${url} to ali-fc region ${region}, code: ${rawData.statusCode}.`,
|
||||
"ALICLOUD_PROXY_ERR",
|
||||
@ -278,7 +295,7 @@ class NetworkDelegate {
|
||||
}
|
||||
}
|
||||
|
||||
const networkDelegate = new NetworkDelegate();
|
||||
const netScheduler = new NetScheduler();
|
||||
const videoInfoRateLimiterConfig: RateLimiterConfig[] = [
|
||||
{
|
||||
window: new SlidingWindow(redis, 0.3),
|
||||
@ -352,14 +369,14 @@ but both should come after addProxy and addTask to ensure proper setup and depen
|
||||
*/
|
||||
|
||||
const regions = ["shanghai", "hangzhou", "qingdao", "beijing", "zhangjiakou", "chengdu", "shenzhen", "hohhot"];
|
||||
networkDelegate.addProxy("native", "native", "");
|
||||
netScheduler.addProxy("native", "native", "");
|
||||
for (const region of regions) {
|
||||
networkDelegate.addProxy(`alicloud-${region}`, "alicloud-fc", region);
|
||||
netScheduler.addProxy(`alicloud-${region}`, "alicloud-fc", region);
|
||||
}
|
||||
networkDelegate.addTask("getVideoInfo", "bilibili", "all");
|
||||
networkDelegate.addTask("getLatestVideos", "bilibili", "all");
|
||||
networkDelegate.addTask("snapshotMilestoneVideo", "bilibili", regions.map((region) => `alicloud-${region}`));
|
||||
networkDelegate.addTask("snapshotVideo", "bili_test", [
|
||||
netScheduler.addTask("getVideoInfo", "bilibili", "all");
|
||||
netScheduler.addTask("getLatestVideos", "bilibili", "all");
|
||||
netScheduler.addTask("snapshotMilestoneVideo", "bilibili", regions.map((region) => `alicloud-${region}`));
|
||||
netScheduler.addTask("snapshotVideo", "bili_test", [
|
||||
"alicloud-qingdao",
|
||||
"alicloud-shanghai",
|
||||
"alicloud-zhangjiakou",
|
||||
@ -367,7 +384,7 @@ networkDelegate.addTask("snapshotVideo", "bili_test", [
|
||||
"alicloud-shenzhen",
|
||||
"alicloud-hohhot",
|
||||
]);
|
||||
networkDelegate.addTask("bulkSnapshot", "bili_strict", [
|
||||
netScheduler.addTask("bulkSnapshot", "bili_strict", [
|
||||
"alicloud-qingdao",
|
||||
"alicloud-shanghai",
|
||||
"alicloud-zhangjiakou",
|
||||
@ -375,13 +392,13 @@ networkDelegate.addTask("bulkSnapshot", "bili_strict", [
|
||||
"alicloud-shenzhen",
|
||||
"alicloud-hohhot",
|
||||
]);
|
||||
networkDelegate.setTaskLimiter("getVideoInfo", videoInfoRateLimiterConfig);
|
||||
networkDelegate.setTaskLimiter("getLatestVideos", null);
|
||||
networkDelegate.setTaskLimiter("snapshotMilestoneVideo", null);
|
||||
networkDelegate.setTaskLimiter("snapshotVideo", null);
|
||||
networkDelegate.setTaskLimiter("bulkSnapshot", null);
|
||||
networkDelegate.setProviderLimiter("bilibili", biliLimiterConfig);
|
||||
networkDelegate.setProviderLimiter("bili_test", bili_test);
|
||||
networkDelegate.setProviderLimiter("bili_strict", bili_strict);
|
||||
netScheduler.setTaskLimiter("getVideoInfo", videoInfoRateLimiterConfig);
|
||||
netScheduler.setTaskLimiter("getLatestVideos", null);
|
||||
netScheduler.setTaskLimiter("snapshotMilestoneVideo", null);
|
||||
netScheduler.setTaskLimiter("snapshotVideo", null);
|
||||
netScheduler.setTaskLimiter("bulkSnapshot", null);
|
||||
netScheduler.setProviderLimiter("bilibili", biliLimiterConfig);
|
||||
netScheduler.setProviderLimiter("bili_test", bili_test);
|
||||
netScheduler.setProviderLimiter("bili_strict", bili_strict);
|
||||
|
||||
export default networkDelegate;
|
||||
export default netScheduler;
|
||||
@ -1,51 +0,0 @@
|
||||
import { findClosestSnapshot, getLatestSnapshot, hasAtLeast2Snapshots } from "db/snapshotSchedule.ts";
|
||||
import { truncate } from "utils/truncate.ts";
|
||||
import { closetMilestone } from "./exec/snapshotTick.ts";
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { HOUR, MINUTE } from "$std/datetime/constants.ts";
|
||||
|
||||
const log = (value: number, base: number = 10) => Math.log(value) / Math.log(base);
|
||||
|
||||
/*
|
||||
* Returns the minimum ETA in hours for the next snapshot
|
||||
* @param client - Postgres client
|
||||
* @param aid - aid of the video
|
||||
* @returns ETA in hours
|
||||
*/
|
||||
export const getAdjustedShortTermETA = async (client: Client, aid: number) => {
|
||||
const latestSnapshot = await getLatestSnapshot(client, aid);
|
||||
// Immediately dispatch a snapshot if there is no snapshot yet
|
||||
if (!latestSnapshot) return 0;
|
||||
const snapshotsEnough = await hasAtLeast2Snapshots(client, aid);
|
||||
if (!snapshotsEnough) return 0;
|
||||
|
||||
const currentTimestamp = new Date().getTime();
|
||||
const timeIntervals = [3 * MINUTE, 20 * MINUTE, 1 * HOUR, 3 * HOUR, 6 * HOUR, 72 * HOUR];
|
||||
const DELTA = 0.00001;
|
||||
let minETAHours = Infinity;
|
||||
|
||||
for (const timeInterval of timeIntervals) {
|
||||
const date = new Date(currentTimestamp - timeInterval);
|
||||
const snapshot = await findClosestSnapshot(client, aid, date);
|
||||
if (!snapshot) continue;
|
||||
const hoursDiff = (latestSnapshot.created_at - snapshot.created_at) / HOUR;
|
||||
const viewsDiff = latestSnapshot.views - snapshot.views;
|
||||
if (viewsDiff <= 0) continue;
|
||||
const speed = viewsDiff / (hoursDiff + DELTA);
|
||||
const target = closetMilestone(latestSnapshot.views);
|
||||
const viewsToIncrease = target - latestSnapshot.views;
|
||||
const eta = viewsToIncrease / (speed + DELTA);
|
||||
let factor = log(2.97 / log(viewsToIncrease + 1), 1.14);
|
||||
factor = truncate(factor, 3, 100);
|
||||
const adjustedETA = eta / factor;
|
||||
if (adjustedETA < minETAHours) {
|
||||
minETAHours = adjustedETA;
|
||||
}
|
||||
}
|
||||
|
||||
if (isNaN(minETAHours)) {
|
||||
minETAHours = Infinity;
|
||||
}
|
||||
|
||||
return minETAHours;
|
||||
};
|
||||
@ -53,7 +53,7 @@ export async function insertVideoInfo(client: Client, aid: number) {
|
||||
query,
|
||||
[aid, stat.view, stat.danmaku, stat.reply, stat.like, stat.coin, stat.share, stat.favorite],
|
||||
);
|
||||
|
||||
|
||||
logger.log(`Inserted video metadata for aid: ${aid}`, "mq");
|
||||
await ClassifyVideoQueue.add("classifyVideo", { aid });
|
||||
}
|
||||
|
||||
@ -1,19 +1,8 @@
|
||||
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
||||
import { getVideoInfo } from "net/getVideoInfo.ts";
|
||||
import { LatestSnapshotType } from "db/schema.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
|
||||
export interface SnapshotNumber {
|
||||
time: number;
|
||||
views: number;
|
||||
coins: number;
|
||||
likes: number;
|
||||
favorites: number;
|
||||
shares: number;
|
||||
danmakus: number;
|
||||
aid: number;
|
||||
replies: number;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fetch video stats from bilibili API and insert into database
|
||||
* @returns {Promise<number|VideoSnapshot>}
|
||||
@ -28,7 +17,7 @@ export async function insertVideoSnapshot(
|
||||
client: Client,
|
||||
aid: number,
|
||||
task: string,
|
||||
): Promise<number | SnapshotNumber> {
|
||||
): Promise<number | LatestSnapshotType> {
|
||||
const data = await getVideoInfo(aid, task);
|
||||
if (typeof data == "number") {
|
||||
return data;
|
||||
@ -53,7 +42,7 @@ export async function insertVideoSnapshot(
|
||||
|
||||
logger.log(`Taken snapshot for video ${aid}.`, "net", "fn:insertVideoSnapshot");
|
||||
|
||||
return {
|
||||
const snapshot: LatestSnapshotType = {
|
||||
aid,
|
||||
views,
|
||||
danmakus,
|
||||
@ -64,4 +53,6 @@ export async function insertVideoSnapshot(
|
||||
favorites,
|
||||
time,
|
||||
};
|
||||
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import networkDelegate from "@core/net/delegate.ts";
|
||||
import { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d.ts";
|
||||
import netScheduler from "mq/scheduler.ts";
|
||||
import { MediaListInfoData, MediaListInfoResponse } from "net/bilibili.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
|
||||
/*
|
||||
@ -12,11 +12,12 @@ import logger from "log/logger.ts";
|
||||
* - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
|
||||
*/
|
||||
export async function bulkGetVideoStats(aids: number[]): Promise<MediaListInfoData | number> {
|
||||
let url = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
|
||||
const baseURL = `https://api.bilibili.com/medialist/gateway/base/resource/infos?resources=`;
|
||||
let url = baseURL;
|
||||
for (const aid of aids) {
|
||||
url += `${aid}:2,`;
|
||||
}
|
||||
const data = await networkDelegate.request<MediaListInfoResponse>(url, "bulkSnapshot");
|
||||
const data = await netScheduler.request<MediaListInfoResponse>(url, "bulkSnapshot");
|
||||
const errMessage = `Error fetching metadata for aid list: ${aids.join(",")}:`;
|
||||
if (data.code !== 0) {
|
||||
logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfo");
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { VideoListResponse } from "@core/net/bilibili.d.ts";
|
||||
import { VideoListResponse } from "net/bilibili.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
import networkDelegate from "@core/net/delegate.ts";
|
||||
import netScheduler from "mq/scheduler.ts";
|
||||
|
||||
export async function getLatestVideoAids(page: number = 1, pageSize: number = 10): Promise<number[]> {
|
||||
const startFrom = 1 + pageSize * (page - 1);
|
||||
@ -8,7 +8,7 @@ export async function getLatestVideoAids(page: number = 1, pageSize: number = 10
|
||||
const range = `${startFrom}-${endTo}`;
|
||||
const errMessage = `Error fetching latest aid for ${range}:`;
|
||||
const url = `https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=${pageSize}&pn=${page}`;
|
||||
const data = await networkDelegate.request<VideoListResponse>(url, "getLatestVideos");
|
||||
const data = await netScheduler.request<VideoListResponse>(url, "getLatestVideos");
|
||||
if (data.code != 0) {
|
||||
logger.error(errMessage + data.message, "net", "getLastestVideos");
|
||||
return [];
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
import networkDelegate from "@core/net/delegate.ts";
|
||||
import { VideoDetailsData, VideoDetailsResponse } from "@core/net/bilibili.d.ts";
|
||||
import netScheduler from "mq/scheduler.ts";
|
||||
import { VideoDetailsData, VideoDetailsResponse } from "net/bilibili.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
|
||||
export async function getVideoDetails(aid: number): Promise<VideoDetailsData | null> {
|
||||
const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`;
|
||||
const data = await networkDelegate.request<VideoDetailsResponse>(url, "getVideoInfo");
|
||||
const data = await netScheduler.request<VideoDetailsResponse>(url, "getVideoInfo");
|
||||
const errMessage = `Error fetching metadata for ${aid}:`;
|
||||
if (data.code !== 0) {
|
||||
logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfo");
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import networkDelegate from "@core/net/delegate.ts";
|
||||
import { VideoInfoData, VideoInfoResponse } from "@core/net/bilibili.d.ts";
|
||||
import netScheduler from "mq/scheduler.ts";
|
||||
import { VideoInfoData, VideoInfoResponse } from "net/bilibili.d.ts";
|
||||
import logger from "log/logger.ts";
|
||||
|
||||
/*
|
||||
@ -17,7 +17,7 @@ import logger from "log/logger.ts";
|
||||
*/
|
||||
export async function getVideoInfo(aid: number, task: string): Promise<VideoInfoData | number> {
|
||||
const url = `https://api.bilibili.com/x/web-interface/view?aid=${aid}`;
|
||||
const data = await networkDelegate.request<VideoInfoResponse>(url, task);
|
||||
const data = await netScheduler.request<VideoInfoResponse>(url, task);
|
||||
const errMessage = `Error fetching metadata for ${aid}:`;
|
||||
if (data.code !== 0) {
|
||||
logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfo");
|
||||
@ -25,27 +25,3 @@ export async function getVideoInfo(aid: number, task: string): Promise<VideoInfo
|
||||
}
|
||||
return data.data;
|
||||
}
|
||||
|
||||
/*
|
||||
* Fetch video metadata from bilibili API by BVID
|
||||
* @param {string} bvid - The video's BVID
|
||||
* @param {string} task - The task name used in scheduler. It can be one of the following:
|
||||
* - snapshotVideo
|
||||
* - getVideoInfo
|
||||
* - snapshotMilestoneVideo
|
||||
* @returns {Promise<VideoInfoData | number>} VideoInfoData or the error code returned by bilibili API
|
||||
* @throws {NetSchedulerError} - The error will be thrown in following cases:
|
||||
* - No proxy is available currently: with error code `NO_PROXY_AVAILABLE`
|
||||
* - The native `fetch` function threw an error: with error code `FETCH_ERROR`
|
||||
* - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR`
|
||||
*/
|
||||
export async function getVideoInfoByBV(bvid: string, task: string): Promise<VideoInfoData | number> {
|
||||
const url = `https://api.bilibili.com/x/web-interface/view?bvid=${bvid}`;
|
||||
const data = await networkDelegate.request<VideoInfoResponse>(url, task);
|
||||
const errMessage = `Error fetching metadata for ${bvid}:`;
|
||||
if (data.code !== 0) {
|
||||
logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfoByBV");
|
||||
return data.code;
|
||||
}
|
||||
return data.data;
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
import { ConnectionOptions, Job, Worker } from "bullmq";
|
||||
import { redis } from "../../core/db/redis.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
import logger from "log/logger.ts";
|
||||
import { classifyVideosWorker, classifyVideoWorker } from "mq/exec/classifyVideo.ts";
|
||||
import { WorkerError } from "mq/schema.ts";
|
||||
@ -18,7 +18,7 @@ Deno.addSignalListener("SIGTERM", async () => {
|
||||
Deno.exit();
|
||||
});
|
||||
|
||||
await Akari.init();
|
||||
Akari.init();
|
||||
|
||||
const filterWorker = new Worker(
|
||||
"classifyVideo",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { ConnectionOptions, Job, Worker } from "bullmq";
|
||||
import { collectSongsWorker, getLatestVideosWorker } from "mq/executors.ts";
|
||||
import { redis } from "../../core/db/redis.ts";
|
||||
import { redis } from "db/redis.ts";
|
||||
import logger from "log/logger.ts";
|
||||
import { lockManager } from "mq/lockManager.ts";
|
||||
import { WorkerError } from "mq/schema.ts";
|
||||
|
||||
@ -19,6 +19,6 @@ export default defineConfig({
|
||||
allow: [".", "../../"],
|
||||
},
|
||||
},
|
||||
plugins: [tsconfigPaths()],
|
||||
plugins: [tsconfigPaths()]
|
||||
},
|
||||
});
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
"": {
|
||||
"name": "frontend",
|
||||
"dependencies": {
|
||||
"@astrojs/node": "^9.1.3",
|
||||
"@astrojs/svelte": "^7.0.9",
|
||||
"@astrojs/tailwind": "^6.0.2",
|
||||
"argon2id": "^1.0.1",
|
||||
"astro": "^5.5.5",
|
||||
@ -33,12 +31,8 @@
|
||||
|
||||
"@astrojs/markdown-remark": ["@astrojs/markdown-remark@6.3.1", "", { "dependencies": { "@astrojs/internal-helpers": "0.6.1", "@astrojs/prism": "3.2.0", "github-slugger": "^2.0.0", "hast-util-from-html": "^2.0.3", "hast-util-to-text": "^4.0.2", "import-meta-resolve": "^4.1.0", "js-yaml": "^4.1.0", "mdast-util-definitions": "^6.0.0", "rehype-raw": "^7.0.0", "rehype-stringify": "^10.0.1", "remark-gfm": "^4.0.1", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.1", "remark-smartypants": "^3.0.2", "shiki": "^3.0.0", "smol-toml": "^1.3.1", "unified": "^11.0.5", "unist-util-remove-position": "^5.0.0", "unist-util-visit": "^5.0.0", "unist-util-visit-parents": "^6.0.1", "vfile": "^6.0.3" } }, "sha512-c5F5gGrkczUaTVgmMW9g1YMJGzOtRvjjhw6IfGuxarM6ct09MpwysP10US729dy07gg8y+ofVifezvP3BNsWZg=="],
|
||||
|
||||
"@astrojs/node": ["@astrojs/node@9.1.3", "", { "dependencies": { "@astrojs/internal-helpers": "0.6.1", "send": "^1.1.0", "server-destroy": "^1.0.1" }, "peerDependencies": { "astro": "^5.3.0" } }, "sha512-YcVxEmeZU8khNdrPYNPN3j//4tYPM+Pw6CthAJ6VE/bw65qEX7ErMRApalY2tibc3YhCeHMmsO9rXGhyW0NNyA=="],
|
||||
|
||||
"@astrojs/prism": ["@astrojs/prism@3.2.0", "", { "dependencies": { "prismjs": "^1.29.0" } }, "sha512-GilTHKGCW6HMq7y3BUv9Ac7GMe/MO9gi9GW62GzKtth0SwukCu/qp2wLiGpEujhY+VVhaG9v7kv/5vFzvf4NYw=="],
|
||||
|
||||
"@astrojs/svelte": ["@astrojs/svelte@7.0.9", "", { "dependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.3", "svelte2tsx": "^0.7.35", "vite": "^6.2.4" }, "peerDependencies": { "astro": "^5.0.0", "svelte": "^5.1.16", "typescript": "^5.3.3" } }, "sha512-EpJfDh7eelYEj/zSwgSHdqJCx6YjiZmpVDEiNjxhnrBwM6Ll7hjllTrNQyfnv7KgJwaVo2SOSz6d1MwV52/T/w=="],
|
||||
|
||||
"@astrojs/tailwind": ["@astrojs/tailwind@6.0.2", "", { "dependencies": { "autoprefixer": "^10.4.21", "postcss": "^8.5.3", "postcss-load-config": "^4.0.2" }, "peerDependencies": { "astro": "^3.0.0 || ^4.0.0 || ^5.0.0", "tailwindcss": "^3.0.24" } }, "sha512-j3mhLNeugZq6A8dMNXVarUa8K6X9AW+QHU9u3lKNrPLMHhOQ0S7VeWhHwEeJFpEK1BTKEUY1U78VQv2gN6hNGg=="],
|
||||
|
||||
"@astrojs/telemetry": ["@astrojs/telemetry@3.2.0", "", { "dependencies": { "ci-info": "^4.1.0", "debug": "^4.3.7", "dlv": "^1.1.3", "dset": "^3.1.4", "is-docker": "^3.0.0", "is-wsl": "^3.1.0", "which-pm-runs": "^1.1.0" } }, "sha512-wxhSKRfKugLwLlr4OFfcqovk+LIFtKwLyGPqMsv+9/ibqqnW3Gv7tBhtKEb0gAyUAC4G9BTVQeQahqnQAhd6IQ=="],
|
||||
@ -223,10 +217,6 @@
|
||||
|
||||
"@sveltejs/acorn-typescript": ["@sveltejs/acorn-typescript@1.0.5", "", { "peerDependencies": { "acorn": "^8.9.0" } }, "sha512-IwQk4yfwLdibDlrXVE04jTZYlLnwsTT2PIOQQGNLWfjavGifnk1JD1LcZjZaBTRcxZu2FfPfNLOE04DSu9lqtQ=="],
|
||||
|
||||
"@sveltejs/vite-plugin-svelte": ["@sveltejs/vite-plugin-svelte@5.0.3", "", { "dependencies": { "@sveltejs/vite-plugin-svelte-inspector": "^4.0.1", "debug": "^4.4.0", "deepmerge": "^4.3.1", "kleur": "^4.1.5", "magic-string": "^0.30.15", "vitefu": "^1.0.4" }, "peerDependencies": { "svelte": "^5.0.0", "vite": "^6.0.0" } }, "sha512-MCFS6CrQDu1yGwspm4qtli0e63vaPCehf6V7pIMP15AsWgMKrqDGCPFF/0kn4SP0ii4aySu4Pa62+fIRGFMjgw=="],
|
||||
|
||||
"@sveltejs/vite-plugin-svelte-inspector": ["@sveltejs/vite-plugin-svelte-inspector@4.0.1", "", { "dependencies": { "debug": "^4.3.7" }, "peerDependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.0", "svelte": "^5.0.0", "vite": "^6.0.0" } }, "sha512-J/Nmb2Q2y7mck2hyCX4ckVHcR5tu2J+MtBEQqpDrrgELZ2uvraQcK/ioCV61AqkdXFgriksOKIceDcQmqnGhVw=="],
|
||||
|
||||
"@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="],
|
||||
|
||||
"@types/estree": ["@types/estree@1.0.7", "", {}, "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ=="],
|
||||
@ -343,14 +333,8 @@
|
||||
|
||||
"decode-named-character-reference": ["decode-named-character-reference@1.1.0", "", { "dependencies": { "character-entities": "^2.0.0" } }, "sha512-Wy+JTSbFThEOXQIR2L6mxJvEs+veIzpmqD7ynWxMXGpnk3smkHQOp6forLdHsKpAMW9iJpaBBIxz285t1n1C3w=="],
|
||||
|
||||
"dedent-js": ["dedent-js@1.0.1", "", {}, "sha512-OUepMozQULMLUmhxS95Vudo0jb0UchLimi3+pQ2plj61Fcy8axbP9hbiD4Sz6DPqn6XG3kfmziVfQ1rSys5AJQ=="],
|
||||
|
||||
"deepmerge": ["deepmerge@4.3.1", "", {}, "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A=="],
|
||||
|
||||
"defu": ["defu@6.1.4", "", {}, "sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg=="],
|
||||
|
||||
"depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="],
|
||||
|
||||
"dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="],
|
||||
|
||||
"destr": ["destr@2.0.5", "", {}, "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA=="],
|
||||
@ -373,16 +357,12 @@
|
||||
|
||||
"eastasianwidth": ["eastasianwidth@0.2.0", "", {}, "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="],
|
||||
|
||||
"ee-first": ["ee-first@1.1.1", "", {}, "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow=="],
|
||||
|
||||
"electron-to-chromium": ["electron-to-chromium@1.5.132", "", {}, "sha512-QgX9EBvWGmvSRa74zqfnG7+Eno0Ak0vftBll0Pt2/z5b3bEGYL6OUXLgKPtvx73dn3dvwrlyVkjPKRRlhLYTEg=="],
|
||||
|
||||
"emoji-regex": ["emoji-regex@10.4.0", "", {}, "sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw=="],
|
||||
|
||||
"emoji-regex-xs": ["emoji-regex-xs@1.0.0", "", {}, "sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg=="],
|
||||
|
||||
"encodeurl": ["encodeurl@2.0.0", "", {}, "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg=="],
|
||||
|
||||
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
|
||||
|
||||
"es-module-lexer": ["es-module-lexer@1.6.0", "", {}, "sha512-qqnD1yMU6tk/jnaMosogGySTZP8YtUgAffA9nMN+E/rjxcfRQ6IEk7IiozUjgxKoFHBGjTLnrHB/YC45r/59EQ=="],
|
||||
@ -391,8 +371,6 @@
|
||||
|
||||
"escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
|
||||
|
||||
"escape-html": ["escape-html@1.0.3", "", {}, "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow=="],
|
||||
|
||||
"escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="],
|
||||
|
||||
"esm-env": ["esm-env@1.2.2", "", {}, "sha512-Epxrv+Nr/CaL4ZcFGPJIYLWFom+YeV1DqMLHJoEd9SYRxNbaFruBwfEX/kkHUJf55j2+TUbmDcmuilbP1TmXHA=="],
|
||||
@ -401,8 +379,6 @@
|
||||
|
||||
"estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="],
|
||||
|
||||
"etag": ["etag@1.8.1", "", {}, "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="],
|
||||
|
||||
"eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="],
|
||||
|
||||
"extend": ["extend@3.0.2", "", {}, "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g=="],
|
||||
@ -421,8 +397,6 @@
|
||||
|
||||
"fraction.js": ["fraction.js@4.3.7", "", {}, "sha512-ZsDfxO51wGAXREY55a7la9LScWpwv9RxIrYABrlvOFBlH/ShPnrtsXeuUIfXKKOVicNxQ+o8JTbJvjS4M89yew=="],
|
||||
|
||||
"fresh": ["fresh@2.0.0", "", {}, "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
@ -467,12 +441,8 @@
|
||||
|
||||
"http-cache-semantics": ["http-cache-semantics@4.1.1", "", {}, "sha512-er295DKPVsV82j5kw1Gjt+ADA/XYHsajl82cGNQG2eyoPkvgUhX+nDIyelzhIWbbsXP39EHcI6l5tYs2FYqYXQ=="],
|
||||
|
||||
"http-errors": ["http-errors@2.0.0", "", { "dependencies": { "depd": "2.0.0", "inherits": "2.0.4", "setprototypeof": "1.2.0", "statuses": "2.0.1", "toidentifier": "1.0.1" } }, "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ=="],
|
||||
|
||||
"import-meta-resolve": ["import-meta-resolve@4.1.0", "", {}, "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw=="],
|
||||
|
||||
"inherits": ["inherits@2.0.4", "", {}, "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="],
|
||||
|
||||
"iron-webcrypto": ["iron-webcrypto@1.2.1", "", {}, "sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg=="],
|
||||
|
||||
"is-arrayish": ["is-arrayish@0.3.2", "", {}, "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ=="],
|
||||
@ -517,8 +487,6 @@
|
||||
|
||||
"longest-streak": ["longest-streak@3.1.0", "", {}, "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g=="],
|
||||
|
||||
"lower-case": ["lower-case@2.0.2", "", { "dependencies": { "tslib": "^2.0.3" } }, "sha512-7fm3l3NAF9WfN6W3JOmf5drwpVqX78JtoGJ3A6W0a6ZnldM41w2fV5D490psKFTpMds8TJse/eHLFFsNHHjHgg=="],
|
||||
|
||||
"lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="],
|
||||
|
||||
"magic-string": ["magic-string@0.30.17", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.0" } }, "sha512-sNPKHvyjVf7gyjwS4xGTaW/mCnF8wnjtifKBEhxfZ7E/S8tQ0rssrwGNn6q8JH/ohItJfSQp9mBtQYuTlH5QnA=="],
|
||||
@ -613,10 +581,6 @@
|
||||
|
||||
"micromatch": ["micromatch@4.0.8", "", { "dependencies": { "braces": "^3.0.3", "picomatch": "^2.3.1" } }, "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA=="],
|
||||
|
||||
"mime-db": ["mime-db@1.54.0", "", {}, "sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ=="],
|
||||
|
||||
"mime-types": ["mime-types@3.0.1", "", { "dependencies": { "mime-db": "^1.54.0" } }, "sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA=="],
|
||||
|
||||
"minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="],
|
||||
|
||||
"minipass": ["minipass@7.1.2", "", {}, "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw=="],
|
||||
@ -633,8 +597,6 @@
|
||||
|
||||
"nlcst-to-string": ["nlcst-to-string@4.0.0", "", { "dependencies": { "@types/nlcst": "^2.0.0" } }, "sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA=="],
|
||||
|
||||
"no-case": ["no-case@3.0.4", "", { "dependencies": { "lower-case": "^2.0.2", "tslib": "^2.0.3" } }, "sha512-fgAN3jGAh+RoxUGZHTSOLJIqUc2wmoBwGR4tbpNAKmmovFoWq0OdRkb0VkldReO2a2iBT/OEulG9XSUc10r3zg=="],
|
||||
|
||||
"node-fetch-native": ["node-fetch-native@1.6.6", "", {}, "sha512-8Mc2HhqPdlIfedsuZoc3yioPuzp6b+L5jRCRY1QzuWZh2EGJVQrGppC6V6cF0bLdbW0+O2YpqCA25aF/1lvipQ=="],
|
||||
|
||||
"node-mock-http": ["node-mock-http@1.0.0", "", {}, "sha512-0uGYQ1WQL1M5kKvGRXWQ3uZCHtLTO8hln3oBjIusM75WoesZ909uQJs/Hb946i2SS+Gsrhkaa6iAO17jRIv6DQ=="],
|
||||
@ -653,8 +615,6 @@
|
||||
|
||||
"ofetch": ["ofetch@1.4.1", "", { "dependencies": { "destr": "^2.0.3", "node-fetch-native": "^1.6.4", "ufo": "^1.5.4" } }, "sha512-QZj2DfGplQAr2oj9KzceK9Hwz6Whxazmn85yYeVuS3u9XTMOGMRx0kO95MQ+vLsj/S/NwBDMMLU5hpxvI6Tklw=="],
|
||||
|
||||
"on-finished": ["on-finished@2.4.1", "", { "dependencies": { "ee-first": "1.1.1" } }, "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg=="],
|
||||
|
||||
"oniguruma-parser": ["oniguruma-parser@0.5.4", "", {}, "sha512-yNxcQ8sKvURiTwP0mV6bLQCYE7NKfKRRWunhbZnXgxSmB1OXa1lHrN3o4DZd+0Si0kU5blidK7BcROO8qv5TZA=="],
|
||||
|
||||
"oniguruma-to-es": ["oniguruma-to-es@4.1.0", "", { "dependencies": { "emoji-regex-xs": "^1.0.0", "oniguruma-parser": "^0.5.4", "regex": "^6.0.1", "regex-recursion": "^6.0.2" } }, "sha512-SNwG909cSLo4vPyyPbU/VJkEc9WOXqu2ycBlfd1UCXLqk1IijcQktSBb2yRQ2UFPsDhpkaf+C1dtT3PkLK/yWA=="],
|
||||
@ -673,8 +633,6 @@
|
||||
|
||||
"parse5": ["parse5@7.2.1", "", { "dependencies": { "entities": "^4.5.0" } }, "sha512-BuBYQYlv1ckiPdQi/ohiivi9Sagc9JG+Ozs0r7b/0iK3sKmrb0b9FdWdBbOdx6hBCM/F9Ir82ofnBhtZOjCRPQ=="],
|
||||
|
||||
"pascal-case": ["pascal-case@3.1.2", "", { "dependencies": { "no-case": "^3.0.4", "tslib": "^2.0.3" } }, "sha512-uWlGT3YSnK9x3BQJaOdcZwrnV6hPpd8jFH1/ucpiLRPh/2zCVJKS19E4GvYHvaCcACn3foXZ0cLB9Wrx1KGe5g=="],
|
||||
|
||||
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
||||
|
||||
"path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="],
|
||||
@ -741,8 +699,6 @@
|
||||
|
||||
"radix3": ["radix3@1.1.2", "", {}, "sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA=="],
|
||||
|
||||
"range-parser": ["range-parser@1.2.1", "", {}, "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg=="],
|
||||
|
||||
"read-cache": ["read-cache@1.0.0", "", { "dependencies": { "pify": "^2.3.0" } }, "sha512-Owdv/Ft7IjOgm/i0xvNDZ1LrRANRfew4b2prF3OWMQLxLfu3bS8FVhCsrSCMK4lR56Y9ya+AThoTpDCTxCmpRA=="],
|
||||
|
||||
"readdirp": ["readdirp@3.6.0", "", { "dependencies": { "picomatch": "^2.2.1" } }, "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA=="],
|
||||
@ -789,12 +745,6 @@
|
||||
|
||||
"semver": ["semver@7.7.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA=="],
|
||||
|
||||
"send": ["send@1.2.0", "", { "dependencies": { "debug": "^4.3.5", "encodeurl": "^2.0.0", "escape-html": "^1.0.3", "etag": "^1.8.1", "fresh": "^2.0.0", "http-errors": "^2.0.0", "mime-types": "^3.0.1", "ms": "^2.1.3", "on-finished": "^2.4.1", "range-parser": "^1.2.1", "statuses": "^2.0.1" } }, "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw=="],
|
||||
|
||||
"server-destroy": ["server-destroy@1.0.1", "", {}, "sha512-rb+9B5YBIEzYcD6x2VKidaa+cqYBJQKnU4oe4E3ANwRRN56yk/ua1YCJT1n21NTS8w6CcOclAKNP3PhdCXKYtQ=="],
|
||||
|
||||
"setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="],
|
||||
|
||||
"sharp": ["sharp@0.33.5", "", { "dependencies": { "color": "^4.2.3", "detect-libc": "^2.0.3", "semver": "^7.6.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.33.5", "@img/sharp-darwin-x64": "0.33.5", "@img/sharp-libvips-darwin-arm64": "1.0.4", "@img/sharp-libvips-darwin-x64": "1.0.4", "@img/sharp-libvips-linux-arm": "1.0.5", "@img/sharp-libvips-linux-arm64": "1.0.4", "@img/sharp-libvips-linux-s390x": "1.0.4", "@img/sharp-libvips-linux-x64": "1.0.4", "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", "@img/sharp-libvips-linuxmusl-x64": "1.0.4", "@img/sharp-linux-arm": "0.33.5", "@img/sharp-linux-arm64": "0.33.5", "@img/sharp-linux-s390x": "0.33.5", "@img/sharp-linux-x64": "0.33.5", "@img/sharp-linuxmusl-arm64": "0.33.5", "@img/sharp-linuxmusl-x64": "0.33.5", "@img/sharp-wasm32": "0.33.5", "@img/sharp-win32-ia32": "0.33.5", "@img/sharp-win32-x64": "0.33.5" } }, "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw=="],
|
||||
|
||||
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||
@ -817,8 +767,6 @@
|
||||
|
||||
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
|
||||
|
||||
"statuses": ["statuses@2.0.1", "", {}, "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ=="],
|
||||
|
||||
"string-width": ["string-width@7.2.0", "", { "dependencies": { "emoji-regex": "^10.3.0", "get-east-asian-width": "^1.0.0", "strip-ansi": "^7.1.0" } }, "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ=="],
|
||||
|
||||
"string-width-cjs": ["string-width@4.2.3", "", { "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", "strip-ansi": "^6.0.1" } }, "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g=="],
|
||||
@ -835,8 +783,6 @@
|
||||
|
||||
"svelte": ["svelte@5.25.7", "", { "dependencies": { "@ampproject/remapping": "^2.3.0", "@jridgewell/sourcemap-codec": "^1.5.0", "@sveltejs/acorn-typescript": "^1.0.5", "@types/estree": "^1.0.5", "acorn": "^8.12.1", "aria-query": "^5.3.1", "axobject-query": "^4.1.0", "clsx": "^2.1.1", "esm-env": "^1.2.1", "esrap": "^1.4.6", "is-reference": "^3.0.3", "locate-character": "^3.0.0", "magic-string": "^0.30.11", "zimmerframe": "^1.1.2" } }, "sha512-0fzXbXaKfSvFUs6Wxev2h4CoEhexZotbTF9EJ4+Cg7MHW64ZnZ9+xUedZyEpgj0Tt9HrYGv9aASHkqjn9b/cPw=="],
|
||||
|
||||
"svelte2tsx": ["svelte2tsx@0.7.35", "", { "dependencies": { "dedent-js": "^1.0.1", "pascal-case": "^3.1.1" }, "peerDependencies": { "svelte": "^3.55 || ^4.0.0-next.0 || ^4.0 || ^5.0.0-next.0", "typescript": "^4.9.4 || ^5.0.0" } }, "sha512-z2lnOnrfb5nrlRfFQI8Qdz03xQqMHUfPj0j8l/fQuydrH89cCeN+v9jgDwK9GyMtdTRUkE7Neu9Gh+vfXJAfuQ=="],
|
||||
|
||||
"tailwindcss": ["tailwindcss@3.4.17", "", { "dependencies": { "@alloc/quick-lru": "^5.2.0", "arg": "^5.0.2", "chokidar": "^3.6.0", "didyoumean": "^1.2.2", "dlv": "^1.1.3", "fast-glob": "^3.3.2", "glob-parent": "^6.0.2", "is-glob": "^4.0.3", "jiti": "^1.21.6", "lilconfig": "^3.1.3", "micromatch": "^4.0.8", "normalize-path": "^3.0.0", "object-hash": "^3.0.0", "picocolors": "^1.1.1", "postcss": "^8.4.47", "postcss-import": "^15.1.0", "postcss-js": "^4.0.1", "postcss-load-config": "^4.0.2", "postcss-nested": "^6.2.0", "postcss-selector-parser": "^6.1.2", "resolve": "^1.22.8", "sucrase": "^3.35.0" }, "bin": { "tailwind": "lib/cli.js", "tailwindcss": "lib/cli.js" } }, "sha512-w33E2aCvSDP0tW9RZuNXadXlkHXqFzSkQew/aIa2i/Sj8fThxwovwlXHSPXTbAHwEIhBFXAedUhP2tueAKP8Og=="],
|
||||
|
||||
"thenify": ["thenify@3.3.1", "", { "dependencies": { "any-promise": "^1.0.0" } }, "sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw=="],
|
||||
@ -849,8 +795,6 @@
|
||||
|
||||
"to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="],
|
||||
|
||||
"toidentifier": ["toidentifier@1.0.1", "", {}, "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA=="],
|
||||
|
||||
"trim-lines": ["trim-lines@3.0.1", "", {}, "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg=="],
|
||||
|
||||
"trough": ["trough@2.2.0", "", {}, "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw=="],
|
||||
|
||||
@ -9,8 +9,6 @@
|
||||
"astro": "astro"
|
||||
},
|
||||
"dependencies": {
|
||||
"@astrojs/node": "^9.1.3",
|
||||
"@astrojs/svelte": "^7.0.9",
|
||||
"@astrojs/tailwind": "^6.0.2",
|
||||
"argon2id": "^1.0.1",
|
||||
"astro": "^5.5.5",
|
||||
|
||||
@ -1,25 +1,13 @@
|
||||
const N_1024 = BigInt(
|
||||
"129023318876534346704360951712586568674758913224876821534686030409476129469193481910786173836188085930974906857867802234113909470848523288588793477904039083513378341278558405407018889387577114155572311708428733260891448259786041525189132461448841652472631435226032063278124857443496954605482776113964107326943",
|
||||
);
|
||||
const N_1024 = BigInt("129023318876534346704360951712586568674758913224876821534686030409476129469193481910786173836188085930974906857867802234113909470848523288588793477904039083513378341278558405407018889387577114155572311708428733260891448259786041525189132461448841652472631435226032063278124857443496954605482776113964107326943")
|
||||
|
||||
const N_2048 = BigInt(
|
||||
"23987552118069940970878653610463005981599204778388399885550631951871084945075866571231062435627294546200946516668493107358732376187241747090707087544153108117326163500579370560400058549184722138636116585329496684877258304519458316233517215780035360354808658620079068489084797380781488445517430961701007542207001544091884001098497324624368085682074645221148086075871342544591022944384890014176612259729018968864426602901247715051556212559854689574013699665035317257438297910516976812428036717668766321871780963854649899276251822244719887233041422346429752896925499321431273560130952088238625622570366815755926694833109",
|
||||
);
|
||||
const N_2048 = BigInt("23987552118069940970878653610463005981599204778388399885550631951871084945075866571231062435627294546200946516668493107358732376187241747090707087544153108117326163500579370560400058549184722138636116585329496684877258304519458316233517215780035360354808658620079068489084797380781488445517430961701007542207001544091884001098497324624368085682074645221148086075871342544591022944384890014176612259729018968864426602901247715051556212559854689574013699665035317257438297910516976812428036717668766321871780963854649899276251822244719887233041422346429752896925499321431273560130952088238625622570366815755926694833109")
|
||||
|
||||
const N_1792 = BigInt(
|
||||
"23987552118069940970878653610463005981599204778388399885550631951871084945075866571231062435627294546200946516668493107358732376187241747090707087544153108117326163500579370560400058549184722138636116585329496684877258304519458316233517215780035360354808658620079068489084797380781488445517430961701007542207001544091884001098497324624368085682074645221148086075871342544591022944384890014176612259729018968864426602901247715051556212559854689574013699665035317257438297910516976812428036717668766321871780963854649899276251822244719887233041422346429752896925499321431273560130952088238625622570366815755926694833109",
|
||||
);
|
||||
const N_1792 = BigInt("23987552118069940970878653610463005981599204778388399885550631951871084945075866571231062435627294546200946516668493107358732376187241747090707087544153108117326163500579370560400058549184722138636116585329496684877258304519458316233517215780035360354808658620079068489084797380781488445517430961701007542207001544091884001098497324624368085682074645221148086075871342544591022944384890014176612259729018968864426602901247715051556212559854689574013699665035317257438297910516976812428036717668766321871780963854649899276251822244719887233041422346429752896925499321431273560130952088238625622570366815755926694833109")
|
||||
|
||||
const N_1536 = BigInt(
|
||||
"1694330250214463438908848400950857073137355630337290254958754184668036770489801447652464038218330711288158361242955860326168191830448553710492926795708495297280933502917598985378231124113971732841791156356676046934277122699383776036675381503510992810963611269045078440132744168908318454891211962146563551929591147663448816841024591820348784855441153716551049843185172472891407933214238000452095646085222944171689449292644270516031799660928056315886939284985905227",
|
||||
);
|
||||
const N_1536 = BigInt("1694330250214463438908848400950857073137355630337290254958754184668036770489801447652464038218330711288158361242955860326168191830448553710492926795708495297280933502917598985378231124113971732841791156356676046934277122699383776036675381503510992810963611269045078440132744168908318454891211962146563551929591147663448816841024591820348784855441153716551049843185172472891407933214238000452095646085222944171689449292644270516031799660928056315886939284985905227")
|
||||
|
||||
const N_3072 = BigInt(
|
||||
"4432919939296042464443862503456460073874727648022810391370558006281079088795179408238989283371442564716849343712703672836423961818025813387453469700639513190304802553045342607888612037304066433501317127429264242784608682213025490491212489901736408833027611579294436675682774458141490718959615677971745638214649336218217578937534746160749039668886450447773018369168258067682196337978245372237157696236362344796867228581553446331915147012787367438751646936429739232247148712001806846526947508445039707404287951727838234648917450736371192435665040644040487427986702098273581288935278964444790007953559851323281510927332862225214878776790605026472021669614552481167977412450477230442015077669503312683966631454347169703030544483487968842349634064181183599641180349414682042575010303056241481622837185325228233789954078775053744988023738762706404546546146837242590884760044438874357295029411988267287001033032827035809135092270843",
|
||||
);
|
||||
const N_3072 = BigInt("4432919939296042464443862503456460073874727648022810391370558006281079088795179408238989283371442564716849343712703672836423961818025813387453469700639513190304802553045342607888612037304066433501317127429264242784608682213025490491212489901736408833027611579294436675682774458141490718959615677971745638214649336218217578937534746160749039668886450447773018369168258067682196337978245372237157696236362344796867228581553446331915147012787367438751646936429739232247148712001806846526947508445039707404287951727838234648917450736371192435665040644040487427986702098273581288935278964444790007953559851323281510927332862225214878776790605026472021669614552481167977412450477230442015077669503312683966631454347169703030544483487968842349634064181183599641180349414682042575010303056241481622837185325228233789954078775053744988023738762706404546546146837242590884760044438874357295029411988267287001033032827035809135092270843")
|
||||
|
||||
const N_4096 = BigInt(
|
||||
"703671044356805218391078271512201582198770553281951369783674142891088501340774249238173262580562112786670043634665390581120113644316651934154746357220932310140476300088580654571796404198410555061275065442553506658401183560336140989074165998202690496991174269748740565700402715364422506782445179963440819952745241176450402011121226863984008975377353558155910994380700267903933205531681076494639818328879475919332604951949178075254600102192323286738973253864238076198710173840170988339024438220034106150475640983877458155141500313471699516670799821379238743709125064098477109094533426340852518505385314780319279862586851512004686798362431227795743253799490998475141728082088984359237540124375439664236138519644100625154580910233437864328111620708697941949936338367445851449766581651338876219676721272448769082914348242483068204896479076062102236087066428603930888978596966798402915747531679758905013008059396214343112694563043918465373870648649652122703709658068801764236979191262744515840224548957285182453209028157886219424802426566456408109642062498413592155064289314088837031184200671561102160059065729282902863248815224399131391716503171191977463328439766546574118092303414702384104112719959325482439604572518549918705623086363111",
|
||||
);
|
||||
const N_4096 = BigInt("703671044356805218391078271512201582198770553281951369783674142891088501340774249238173262580562112786670043634665390581120113644316651934154746357220932310140476300088580654571796404198410555061275065442553506658401183560336140989074165998202690496991174269748740565700402715364422506782445179963440819952745241176450402011121226863984008975377353558155910994380700267903933205531681076494639818328879475919332604951949178075254600102192323286738973253864238076198710173840170988339024438220034106150475640983877458155141500313471699516670799821379238743709125064098477109094533426340852518505385314780319279862586851512004686798362431227795743253799490998475141728082088984359237540124375439664236138519644100625154580910233437864328111620708697941949936338367445851449766581651338876219676721272448769082914348242483068204896479076062102236087066428603930888978596966798402915747531679758905013008059396214343112694563043918465373870648649652122703709658068801764236979191262744515840224548957285182453209028157886219424802426566456408109642062498413592155064289314088837031184200671561102160059065729282902863248815224399131391716503171191977463328439766546574118092303414702384104112719959325482439604572518549918705623086363111")
|
||||
|
||||
export const N_ARRAY = [N_1024, N_1536, N_1792, N_2048, N_3072, N_4096];
|
||||
export const N_ARRAY = [N_1024, N_1536, N_1792, N_2048, N_3072, N_4096];
|
||||
@ -33,6 +33,7 @@
|
||||
|
||||
参见[CVSA文档](https://docs.projectcvsa.com/)。
|
||||
|
||||
|
||||
## 开放许可
|
||||
|
||||
受本文以[CC BY-NC-SA 4.0协议](https://creativecommons.org/licenses/by-nc-sa/4.0/)提供。
|
||||
|
||||
Loading…
Reference in New Issue
Block a user