diff --git a/.idea/data_source_mapping.xml b/.idea/data_source_mapping.xml index fe1e0ad..e7530ed 100644 --- a/.idea/data_source_mapping.xml +++ b/.idea/data_source_mapping.xml @@ -9,6 +9,7 @@ + diff --git a/bun.lock b/bun.lock index 9a8bcfc..8def0ae 100644 --- a/bun.lock +++ b/bun.lock @@ -45,7 +45,7 @@ "name": "cf-worker", "version": "0.0.0", "dependencies": { - "@alikia/http-parser": "^1.0.2", + "@alikia/http-parser": "1.0.2", }, "devDependencies": { "@cloudflare/vitest-pool-workers": "^0.8.19", diff --git a/packages/cf-worker/package.json b/packages/cf-worker/package.json index 052631a..3d9a698 100644 --- a/packages/cf-worker/package.json +++ b/packages/cf-worker/package.json @@ -16,6 +16,6 @@ "wrangler": "^4.56.0" }, "dependencies": { - "@alikia/http-parser": "^1.0.2" + "@alikia/http-parser": "1.0.2" } } diff --git a/packages/cf-worker/src/index.ts b/packages/cf-worker/src/index.ts index b166e4f..5336f53 100644 --- a/packages/cf-worker/src/index.ts +++ b/packages/cf-worker/src/index.ts @@ -23,19 +23,11 @@ interface ProxyRequest { headers?: Record; } -interface ParsedHeaders { - status: number; - statusText: string; - headers: Headers; - headerEnd: number; -} - const CONFIG: ProxyConfig = { TIMEOUT_MS: 5000, }; const encoder = new TextEncoder(); -const decoder = new TextDecoder(); function concatUint8Arrays(...arrays: Uint8Array[]): Uint8Array { const total = arrays.reduce((sum, arr) => sum + arr.length, 0); @@ -48,31 +40,6 @@ function concatUint8Arrays(...arrays: Uint8Array[]): Uint8Array { return result; } -function parseHttpHeaders(buff: Uint8Array): ParsedHeaders | null { - const text = decoder.decode(buff); - const headerEnd = text.indexOf("\r\n\r\n"); - if (headerEnd === -1) return null; - - const lines = text.slice(0, headerEnd).split("\r\n"); - const statusMatch = lines[0].match(/HTTP\/1\.[01] (\d+) (.*)/); - if (!statusMatch) throw new Error("Invalid status line"); - - const headers = new Headers(); - for (let i = 1; i < lines.length; i++) { - const idx = lines[i].indexOf(": "); - if (idx !== -1) { - headers.append(lines[i].slice(0, idx), lines[i].slice(idx + 2)); - } - } - - return { - headerEnd, - headers, - status: Number(statusMatch[1]), - statusText: statusMatch[2], - }; -} - function withTimeout(promise: Promise, ms: number): Promise { return Promise.race([ promise, @@ -198,6 +165,16 @@ function createErrorResponse(message: string, status: number, requestId: string) ); } +function isJSON(str: string) { + if (typeof str !== "string") return false; + try { + const result = JSON.parse(str); + return typeof result === "object" && result !== null; + } catch (e) { + return false; + } +} + export default { async fetch(request: Request, _env: Env, _ctx: ExecutionContext): Promise { const requestId = crypto.randomUUID().slice(0, 8); // Track this specific request @@ -223,12 +200,14 @@ export default { } try { - console.log(`[${requestId}] Attempting handleSocket...`); const data = await withTimeout( handleSocket(targetUrl, customHeaders, requestTime), CONFIG.TIMEOUT_MS ); - console.log(`[${requestId}] Success via handleSocket (${Date.now() - requestTime}ms)`); + const json = isJSON(data.data); + console.log( + `[${requestId}] Success via handleSocket (${Date.now() - requestTime}ms) ${data.data.length} bytes, isJSON: ${json}, rawData: ${data.data}` + ); return createJsonResponse(data, requestId); } catch (socketErr: any) { console.warn( diff --git a/packages/core/log/index.ts b/packages/core/log/index.ts index 6f7ae59..577671e 100644 --- a/packages/core/log/index.ts +++ b/packages/core/log/index.ts @@ -2,46 +2,86 @@ import chalk from "chalk"; import type { TransformableInfo } from "logform"; import winston, { format, transports } from "winston"; -const customFormat = format.printf((info: TransformableInfo) => { - const { timestamp, level, message, service, codePath, error } = info; +/* ------------------------------------------------- + * Bun-style console formatter + * ------------------------------------------------- */ +function formatLikeConsole(input: unknown, colors: boolean): string { + const inspect = (value: unknown) => + Bun.inspect(value, { + colors, + compact: false, + depth: 6, + }); + + // console.log(...args) + if (Array.isArray(input)) { + return input.map(inspect).join(" "); + } + + if (input instanceof Error) { + const stack = input.stack ?? input.message; + return colors ? chalk.red(stack) : stack; + } + + if (typeof input === "string") { + return input; + } + + return inspect(input); +} + +/* ------------------------------------------------- + * Console format (colored, human-readable) + * ------------------------------------------------- */ +const customConsoleFormat = format.printf((info: TransformableInfo) => { + const { timestamp, level, message, service, codePath } = info; + const coloredService = service ? chalk.magenta(service) : ""; const coloredCodePath = codePath ? chalk.grey(`@${codePath}`) : ""; const colon = service || codePath ? ": " : ""; - const err = error as Error | undefined; - if (err) { - return `${timestamp} [${level}] ${coloredService}${colon}${message}\n${chalk.red(err.stack) ?? ""}`; - } - return coloredCodePath - ? `${timestamp} [${level}] ${coloredService}${coloredCodePath}${colon}${message}` - : `${timestamp} [${level}] ${coloredService}${colon}${message}`; + const renderedMessage = formatLikeConsole(message, true); + + return `${timestamp} [${level}] ${coloredService}${coloredCodePath}${colon}${renderedMessage}`; }); -const timestampFormat = format.timestamp({ format: "YYYY-MM-DD HH:mm:ss.SSSZZ" }); +/* ------------------------------------------------- + * Timestamp + * ------------------------------------------------- */ +const timestampFormat = format.timestamp({ + format: "YYYY-MM-DD HH:mm:ss.SSSZZ", +}); +/* ------------------------------------------------- + * File transport factory (no colors) + * ------------------------------------------------- */ const createTransport = (level: string, filename: string) => { - const MB = 1000000; - let maxsize; - let maxFiles; - let tailable; + const MB = 1_000_000; + let maxsize: number | undefined; + let maxFiles: number | undefined; + const tailable = false; + if (level === "silly") { maxsize = 500 * MB; - maxFiles = undefined; - tailable = false; } else if (level === "warn") { maxsize = 10 * MB; maxFiles = 5; - tailable = false; } - function replacer(key: unknown, value: unknown) { + + function replacer(_: unknown, value: unknown) { if (typeof value === "bigint") { return value.toString(); } - if (key === "error") { - return undefined; + if (value instanceof Error) { + return { + message: value.message, + name: value.name, + stack: value.stack, + }; } return value; } + return new transports.File({ filename, format: format.combine(timestampFormat, format.json({ replacer })), @@ -52,20 +92,21 @@ const createTransport = (level: string, filename: string) => { }); }; +/* ------------------------------------------------- + * Paths + * ------------------------------------------------- */ const sillyLogPath = process.env["LOG_VERBOSE"] ?? "logs/verbose.log"; const warnLogPath = process.env["LOG_WARN"] ?? "logs/warn.log"; const errorLogPath = process.env["LOG_ERROR"] ?? "logs/error.log"; +/* ------------------------------------------------- + * Winston logger + * ------------------------------------------------- */ const winstonLogger = winston.createLogger({ levels: winston.config.npm.levels, transports: [ new transports.Console({ - format: format.combine( - format.timestamp({ format: "YYYY-MM-DD HH:mm:ss.SSSZZ" }), - format.colorize(), - format.errors({ stack: true }), - customFormat - ), + format: format.combine(timestampFormat, format.colorize(), customConsoleFormat), level: "debug", }), createTransport("silly", sillyLogPath), @@ -74,28 +115,36 @@ const winstonLogger = winston.createLogger({ ], }); +/* ------------------------------------------------- + * Public logger API + * ------------------------------------------------- */ const logger = { - debug: (message: string, service?: string, codePath?: string) => { - winstonLogger.debug(message, { codePath, service }); + debug: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.debug(message as string, { codePath, service }); }, - error: (error: string | Error, service?: string, codePath?: string) => { - if (error instanceof Error) { - winstonLogger.error(error.message, { codePath, error: error, service }); - } else { - winstonLogger.error(error, { codePath, service }); - } + + error: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.error(message as string, { codePath, service }); }, - log: (message: string, service?: string, codePath?: string) => { - winstonLogger.info(message, { codePath, service }); + + info: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.info(message as string, { codePath, service }); }, - silly: (message: string, service?: string, codePath?: string) => { - winstonLogger.silly(message, { codePath, service }); + + log: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.info(message as string, { codePath, service }); }, - verbose: (message: string, service?: string, codePath?: string) => { - winstonLogger.verbose(message, { codePath, service }); + + silly: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.silly(message as string, { codePath, service }); }, - warn: (message: string, service?: string, codePath?: string) => { - winstonLogger.warn(message, { codePath, service }); + + verbose: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.verbose(message as string, { codePath, service }); + }, + + warn: (message: unknown, service?: string, codePath?: string) => { + winstonLogger.warn(message as string, { codePath, service }); }, }; diff --git a/packages/core/net/delegate.ts b/packages/core/net/delegate.ts index 6a7a441..98a9715 100644 --- a/packages/core/net/delegate.ts +++ b/packages/core/net/delegate.ts @@ -23,7 +23,7 @@ import { ReplyError } from "ioredis"; type ProxyType = "native" | "alicloud-fc" | "ip-proxy" | "cf-worker"; -const aliRegions = ["hangzhou"] as const; +const aliRegions = ["hangzhou", "beijing", "shanghai", "chengdu"] as const; type AliRegion = (typeof aliRegions)[number]; function createAliProxiesObject(regions: T) { @@ -42,8 +42,8 @@ function createAliProxiesObject(regions: T) { ); } -const _aliProxiesObject = createAliProxiesObject(aliRegions); -const _aliProxies = aliRegions.map((region) => `alicloud_${region}` as `alicloud_${AliRegion}`); +const aliProxiesObject = createAliProxiesObject(aliRegions); +const aliProxies = aliRegions.map((region) => `alicloud_${region}` as `alicloud_${AliRegion}`); const proxies = { "cf-worker": { @@ -56,6 +56,8 @@ const proxies = { data: {}, type: "native" as const, }, + + ...aliProxiesObject, } satisfies Record; interface FCResponse { @@ -152,14 +154,9 @@ bili_normal[0].max = 5; bili_normal[1].max = 40; bili_normal[2].max = 200; -const bili_strict = structuredClone(biliLimiterConfig); -bili_strict[0].max = 1; -bili_strict[1].max = 6; -bili_strict[2].max = 100; - type MyProxyKeys = keyof typeof proxies; -const _fcProxies = aliRegions.map((region) => `alicloud_${region}`) as MyProxyKeys[]; +const fcProxies = aliRegions.map((region) => `alicloud_${region}`) as MyProxyKeys[]; function createNetworkConfig( config: NetworkConfigInternal @@ -169,36 +166,41 @@ function createNetworkConfig { const url = `https://api.bilibili.com/x/web-interface/view/detail?aid=${aid}`; const { data } = await networkDelegate.request(url, "getVideoInfo"); const errMessage = `Error fetching metadata for ${aid}:`; if (data.code !== 0) { - logger.error(errMessage + data.code + "-" + data.message, "net", "fn:getVideoInfo"); + logger.error(`${errMessage + data.code}-${data.message}`, "net", "fn:getVideoInfo"); return null; } return data.data; diff --git a/packages/crawler/mq/exec/getVideoInfo.ts b/packages/crawler/mq/exec/getVideoInfo.ts index 1edcd89..f832a0e 100644 --- a/packages/crawler/mq/exec/getVideoInfo.ts +++ b/packages/crawler/mq/exec/getVideoInfo.ts @@ -1,5 +1,7 @@ import { bilibiliUser, db, videoSnapshot } from "@core/drizzle"; +import { SECOND } from "@core/lib"; import logger from "@core/log"; +import { NetSchedulerError } from "@core/net/delegate"; import { getVideoDetails } from "@core/net/getVideoDetails"; import type { Job } from "bullmq"; import { @@ -9,7 +11,7 @@ import { } from "db/bilibili_metadata"; import { eq } from "drizzle-orm"; import { snapshotCounter } from "metrics"; -import { ClassifyVideoQueue, latestVideosEventsProducer } from "mq/index"; +import { ClassifyVideoQueue, LatestVideosQueue, latestVideosEventsProducer } from "mq/index"; import type { GetVideoInfoJobData } from "mq/schema"; import { insertIntoSongs } from "mq/task/collectSongs"; @@ -46,73 +48,91 @@ export const getVideoInfoWorker = async (job: Job): Promise await publishAddsongEvent(songs[0].id, job.data.uid); return; } - const data = await getVideoDetails(aid); - if (data === null) { - return null; - } + try { + const data = await getVideoDetails(aid); + if (data === null) { + return null; + } - const uid = data.View.owner.mid; + const uid = data.View.owner.mid; - await insertIntoMetadata({ - aid, - bvid: data.View.bvid, - coverUrl: data.View.pic, - description: data.View.desc, - duration: data.View.duration, - publishedAt: new Date(data.View.pubdate * 1000).toISOString(), - tags: data.Tags.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type)) - .map((tag) => tag.tag_name) - .join(","), - title: data.View.title, - uid: uid, - }); - - const userExists = await userExistsInBiliUsers(aid); - if (!userExists) { - await db.insert(bilibiliUser).values({ - avatar: data.View.owner.face, - desc: data.Card.card.sign, - fans: data.Card.follower, - uid, - username: data.View.owner.name, + await insertIntoMetadata({ + aid, + bvid: data.View.bvid, + coverUrl: data.View.pic, + description: data.View.desc, + duration: data.View.duration, + publishedAt: new Date(data.View.pubdate * 1000).toISOString(), + tags: data.Tags.filter((tag) => !["old_channel", "topic"].indexOf(tag.tag_type)) + .map((tag) => tag.tag_name) + .join(","), + title: data.View.title, + uid: uid, }); - } else { - await db - .update(bilibiliUser) - .set({ + + const userExists = await userExistsInBiliUsers(aid); + if (!userExists) { + await db.insert(bilibiliUser).values({ avatar: data.View.owner.face, desc: data.Card.card.sign, fans: data.Card.follower, + uid, username: data.View.owner.name, - }) - .where(eq(bilibiliUser.uid, uid)); + }); + } else { + await db + .update(bilibiliUser) + .set({ + avatar: data.View.owner.face, + desc: data.Card.card.sign, + fans: data.Card.follower, + username: data.View.owner.name, + }) + .where(eq(bilibiliUser.uid, uid)); + } + + const stat = data.View.stat; + + await db.insert(videoSnapshot).values({ + aid, + coins: stat.coin, + danmakus: stat.danmaku, + favorites: stat.favorite, + likes: stat.like, + replies: stat.reply, + shares: stat.share, + views: stat.view, + }); + + snapshotCounter.add(1); + + logger.log(`Inserted video metadata for aid: ${aid}`, "mq"); + + if (!insertSongs) { + await ClassifyVideoQueue.add("classifyVideo", { aid }); + return; + } + const songs = await insertIntoSongs(aid); + if (songs.length === 0) { + logger.warn(`Failed to insert song for aid: ${aid}`, "mq", "fn:getVideoInfoWorker"); + return; + } + await publishAddsongEvent(songs[0].id, job.data.uid); + } catch (e) { + if (e instanceof NetSchedulerError) { + await LatestVideosQueue.add( + "getVideoInfo", + { aid }, + { + attempts: 10, + backoff: { + delay: 30 * SECOND, + jitter: 1, + type: "fixed", + }, + } + ); + } + logger.error(e, "mq", "fn:getVideoInfoWorker"); } - - const stat = data.View.stat; - - await db.insert(videoSnapshot).values({ - aid, - coins: stat.coin, - danmakus: stat.danmaku, - favorites: stat.favorite, - likes: stat.like, - replies: stat.reply, - shares: stat.share, - views: stat.view, - }); - - snapshotCounter.add(1); - - logger.log(`Inserted video metadata for aid: ${aid}`, "mq"); - - if (!insertSongs) { - await ClassifyVideoQueue.add("classifyVideo", { aid }); - return; - } - const songs = await insertIntoSongs(aid); - if (songs.length === 0) { - logger.warn(`Failed to insert song for aid: ${aid}`, "mq", "fn:getVideoInfoWorker"); - return; - } - await publishAddsongEvent(songs[0].id, job.data.uid); }; diff --git a/packages/crawler/net/bulkGetVideoStats.ts b/packages/crawler/net/bulkGetVideoStats.ts index f4f7bb8..29101bc 100644 --- a/packages/crawler/net/bulkGetVideoStats.ts +++ b/packages/crawler/net/bulkGetVideoStats.ts @@ -1,6 +1,6 @@ import logger from "@core/log"; import type { MediaListInfoData, MediaListInfoResponse } from "@core/net/bilibili.d"; -import networkDelegate from "@core/net/delegate"; +import networkDelegate, { type RequestTasks } from "@core/net/delegate"; /* * Bulk fetch video metadata from bilibili API @@ -11,7 +11,10 @@ import networkDelegate from "@core/net/delegate"; * - The native `fetch` function threw an error: with error code `FETCH_ERROR` * - The alicloud-fc threw an error: with error code `ALICLOUD_FC_ERROR` */ -export async function bulkGetVideoStats(aids: number[]): Promise< +export async function bulkGetVideoStats( + aids: number[], + task?: RequestTasks +): Promise< | { data: MediaListInfoData; time: number; @@ -25,7 +28,7 @@ export async function bulkGetVideoStats(aids: number[]): Promise< } const { data, time } = await networkDelegate.request( url, - "bulkSnapshot" + task ?? "bulkSnapshot" ); const errMessage = `Error fetching metadata for aid list: ${aids.join(",")}:`; if (data.code !== 0) { diff --git a/src/annualArchive.ts b/src/annualArchive.ts new file mode 100644 index 0000000..3dcd11e --- /dev/null +++ b/src/annualArchive.ts @@ -0,0 +1,95 @@ +import { bilibiliMetadata, db, eta, type VideoSnapshotType } from "@core/drizzle"; +import { SECOND } from "@core/lib"; +import logger from "@core/log"; +import { NetSchedulerError } from "@core/net/delegate"; +import { bulkGetVideoStats } from "@crawler/net/bulkGetVideoStats"; +import { desc, eq } from "drizzle-orm"; + +const store = Bun.file(`temp/annualSnapshots.json`); + +const snapshots: Omit[] = []; + +if (await store.exists()) { + // load +} + +const aids = await db + .select({ + aid: bilibiliMetadata.aid, + }) + .from(bilibiliMetadata) + .leftJoin(eta, eq(bilibiliMetadata.aid, eta.aid)) + .orderBy(desc(eta.speed)) + .then((rows) => { + const mapped = rows.map((row) => row.aid); + return mapped.filter((item): item is number => item !== null); + }); + +const totalAids = aids.length; + +logger.log(`Total aids: ${totalAids}`); + +const bulkSize = 50; + +const groupedAids = []; +for (let i = 0; i < aids.length; i += bulkSize) { + groupedAids.push(aids.slice(i, i + bulkSize)); +} + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +const serialize = async () => { + const json = JSON.stringify(snapshots, null, 4); + await store.write(json); +}; + +let aidsProcessed = 0; + +const requestForSnapshot = async (aids: number[], depth: number = 0) => { + if (depth > 10) { + logger.error(`Cannot fetch metadata for aids: ${aids.join(",")}, depth: ${depth}`); + return; + } + try { + const rawData = await bulkGetVideoStats(aids, "annualArchive"); + logger.log(`Fetched metadata for ${aids.length} aids, depth: ${depth}`); + if (typeof rawData === "number") { + await sleep(2 * SECOND); + return requestForSnapshot(aids, depth + 1); + } + for (const item of rawData.data) { + snapshots.push({ + aid: item.id, + coins: item.cnt_info.coin, + createdAt: new Date(rawData.time).toISOString(), + danmakus: item.cnt_info.danmaku, + favorites: item.cnt_info.collect, + likes: item.cnt_info.thumb_up, + replies: item.cnt_info.reply, + shares: item.cnt_info.share, + views: item.cnt_info.play, + }); + aidsProcessed += 1; + } + } catch (e) { + if (e instanceof NetSchedulerError) { + requestForSnapshot(aids, 1); + } + } +}; + +const taskFactories = groupedAids.map((group) => () => requestForSnapshot(group)); + +const concurrency = 100; + +for (let i = 0; i < taskFactories.length; i += concurrency) { + const batch = taskFactories.slice(i, i + concurrency); + + await Promise.all(batch.map((factory) => factory())); + + logger.log(`Processed ${aidsProcessed} of ${totalAids}`); + await sleep(1.7 * SECOND); + serialize(); +} +await serialize(); +process.exit(0); diff --git a/src/extractAidsFromEvocalRank.ts b/src/extractAidsFromEvocalRank.ts new file mode 100644 index 0000000..96c462f --- /dev/null +++ b/src/extractAidsFromEvocalRank.ts @@ -0,0 +1,373 @@ +export interface Root { + version: number; + ranknum: number; + url: string; + coverurl: string; + pubdate: string; + generate_time: string; + generate_timestamp: number; + collect_start_time: string; + collect_end_time: string; + collect_start_time_timestamp: number; + collect_end_time_timestamp: number; + main_rank: MainRank[]; + second_rank: SecondRank[]; + super_hit: SuperHit[]; + pick_up: PickUp[]; + oth_pickup: any[]; + Vocaloid_pick_up: VocaloidPickUp[]; + "history-1-year": History1Year[]; + "history-10-year": History10Year[]; + ed: Ed[]; + op: Op[]; + statistic: Statistic; + thanks_list: any[]; +} + +export interface MainRank { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: any; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource; + rank: number; + ext_rank: ExtRank; +} + +export interface ReferSource { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface ExtRank { + vocaloid?: number; +} + +export interface SecondRank { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource2; + rank: number; + ext_rank: ExtRank2; +} + +export interface ReferSource2 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface ExtRank2 { + vocaloid?: number; +} + +export interface SuperHit { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource3; + superHit_times: number; + rank: string; +} + +export interface ReferSource3 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface PickUp { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource4; + rank: number; + ext_rank: ExtRank4; +} + +export interface ReferSource4 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface ExtRank4 { + vocaloid?: number; +} + +export interface VocaloidPickUp { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource5; + rank: number; + ext_rank: ExtRank5; +} + +export interface ReferSource5 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface ExtRank5 { + vocaloid: number; +} + +export interface History1Year { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource6; + rank: number; + ext_rank: ExtRank6; +} + +export interface ReferSource6 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface ExtRank6 { + vocaloid?: number; +} + +export interface History10Year { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; + referSource: ReferSource7; + rank: number; +} + +export interface ReferSource7 { + point: number; + play: number; + coin: number; + comment: number; + danmaku: number; + favorite: number; + like: number; + share: number; +} + +export interface Ed { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; +} + +export interface Op { + url: string; + avid: string; + coverurl: string; + title: string; + pubdate: string; +} + +export interface Statistic { + diff: Diff; + total_collect_count: number; + new_video_count: number; + new_in_rank_count: number; + new_in_mainrank_count: number; + pick_up_count: number; + oth_pick_up_count: number; + new_vc_in_rank_count: number; + new_vc_in_mainrank_count: number; + vc_in_rank_count: number; + vc_in_mainrank_count: number; + new_sv_in_rank_count: number; + new_sv_in_mainrank_count: number; + sv_in_rank_count: number; + sv_in_mainrank_count: number; + new_ace_in_rank_count: number; + new_ace_in_mainrank_count: number; + ace_in_rank_count: number; + ace_in_mainrank_count: number; +} + +export interface Diff { + total_play: number; + new_video_count: number; + new_in_rank_count: number; + new_in_mainrank_count: number; + new_vc_in_rank_count: number; + new_vc_in_mainrank_count: number; + vc_in_rank_count: number; + vc_in_mainrank_count: number; + new_sv_in_rank_count: number; + new_sv_in_mainrank_count: number; + sv_in_rank_count: number; + sv_in_mainrank_count: number; + new_ace_in_rank_count: number; + new_ace_in_mainrank_count: number; + ace_in_rank_count: number; + ace_in_mainrank_count: number; +} + +const aids = new Set(); +const f = Bun.file("evocalrank.json"); + +for (let i = 699; i >= 520; i--) { + const url = `https://www.evocalrank.com/data/rank_data/${i}.json`; + const response = await fetch(url); + const data = await response.json() as Partial; + if (data.main_rank) { + for (const item of data.main_rank) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.second_rank) { + for (const item of data.second_rank) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.pick_up) { + for (const item of data.pick_up) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.super_hit) { + for (const item of data.super_hit) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.Vocaloid_pick_up) { + for (const item of data.Vocaloid_pick_up) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.ed) { + for (const item of data.ed) { + if (item.avid) { + aids.add(item.avid); + } + } + } + if (data.op) { + for (const item of data.op) { + if (item.avid) { + aids.add(item.avid); + } + } + } + + const serialized = JSON.stringify([...aids], null, 4); + await f.write(serialized); + console.log(`${i} ${aids.size}`); +}