From 4b48357ab62f1d09d1d2c4c4553c8e70d72df55a Mon Sep 17 00:00:00 2001 From: alikia2x Date: Fri, 14 Feb 2025 03:06:39 +0800 Subject: [PATCH] update: insert duration as metadata --- lib/db/allData.ts | 20 +++++++++++++------- lib/net/bisectVideoStartFrom.ts | 24 ++++++++++++------------ lib/net/getLatestVideos.ts | 24 ++++-------------------- lib/task/insertLatestVideo.ts | 9 ++++----- 4 files changed, 33 insertions(+), 44 deletions(-) diff --git a/lib/db/allData.ts b/lib/db/allData.ts index 92c225b..ab3f6e6 100644 --- a/lib/db/allData.ts +++ b/lib/db/allData.ts @@ -1,20 +1,26 @@ import { Client, Transaction } from "https://deno.land/x/postgres@v0.19.3/mod.ts"; -import { AllDataType } from "lib/db/schema.d.ts"; import logger from "lib/log/logger.ts"; -import { parseTimestampFromPsql } from "lib/utils/formatTimestampToPostgre.ts"; +import { formatTimestampToPsql, parseTimestampFromPsql } from "lib/utils/formatTimestampToPostgre.ts"; +import { VideoListVideo } from "lib/net/bilibili.d.ts"; +import { HOUR, SECOND } from "$std/datetime/constants.ts"; export async function videoExistsInAllData(client: Client, aid: number) { return await client.queryObject<{ exists: boolean }>(`SELECT EXISTS(SELECT 1 FROM all_data WHERE aid = $1)`, [aid]) .then((result) => result.rows[0].exists); } -export async function insertIntoAllData(client: Client, data: AllDataType) { +export async function biliUserExists(client: Client, uid: number) { + return await client.queryObject<{ exists: boolean }>(`SELECT EXISTS(SELECT 1 FROM bili_user WHERE uid = $1)`, [uid]) + .then((result) => result.rows[0].exists); +} + +export async function insertIntoAllData(client: Client, data: VideoListVideo) { logger.log(`inserted ${data.aid}`, "db-all_data"); - return await client.queryObject( - `INSERT INTO all_data (aid, bvid, description, uid, tags, title, published_at) - VALUES ($1, $2, $3, $4, $5, $6, $7) + await client.queryObject( + `INSERT INTO all_data (aid, bvid, description, uid, tags, title, published_at, duration) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT (aid) DO NOTHING`, - [data.aid, data.bvid, data.description, data.uid, data.tags, data.title, data.published_at], + [data.aid, data.bvid, data.desc, data.owner.mid, null, data.title, formatTimestampToPsql(data.pubdate * SECOND + 8 * HOUR), data.duration], ); } diff --git a/lib/net/bisectVideoStartFrom.ts b/lib/net/bisectVideoStartFrom.ts index d663e6c..9121504 100644 --- a/lib/net/bisectVideoStartFrom.ts +++ b/lib/net/bisectVideoStartFrom.ts @@ -1,7 +1,8 @@ import { getLatestVideos } from "lib/net/getLatestVideos.ts"; -import { AllDataType } from "lib/db/schema.d.ts"; +import { HOUR, SECOND } from "$std/datetime/constants.ts"; +import { VideoListVideo } from "lib/net/bilibili.d.ts"; -export async function getVideoPositionInNewList(timestamp: number): Promise { +export async function getVideoPositionInNewList(timestamp: number): Promise { const virtualPageSize = 50; let lowPage = 1; @@ -10,16 +11,15 @@ export async function getVideoPositionInNewList(timestamp: number): Promise timestamp) { lo = mid + 1; } else { @@ -63,15 +63,15 @@ export async function getVideoPositionInNewList(timestamp: number): Promise 0) { for (let i = 0; i < boundaryVideos.length; i++) { const video = boundaryVideos[i]; - if (!video.published_at) { + if (!video.pubdate) { continue; } - const videoTime = Date.parse(video.published_at); + const videoTime = video.pubdate * SECOND + 8 * HOUR; if (videoTime > timestamp) { indexInPage++; } else { diff --git a/lib/net/getLatestVideos.ts b/lib/net/getLatestVideos.ts index 33b539c..a3db732 100644 --- a/lib/net/getLatestVideos.ts +++ b/lib/net/getLatestVideos.ts @@ -1,15 +1,10 @@ -import { VideoListResponse } from "lib/net/bilibili.d.ts"; -import { formatTimestampToPsql as formatPublishedAt } from "lib/utils/formatTimestampToPostgre.ts"; -import { AllDataType } from "lib/db/schema.d.ts"; +import { VideoListResponse, VideoListVideo } from "lib/net/bilibili.d.ts"; import logger from "lib/log/logger.ts"; -import { HOUR, SECOND } from "$std/datetime/constants.ts"; export async function getLatestVideos( page: number = 1, - pageSize: number = 10, - sleepRate: number = 250, - fetchTags: boolean = true, -): Promise { + pageSize: number = 10 +): Promise { try { const response = await fetch( `https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=${pageSize}&pn=${page}`, @@ -26,18 +21,7 @@ export async function getLatestVideos( return []; } - return data.data.archives.map((video) => { - const published_at = formatPublishedAt(video.pubdate * SECOND + 8 * HOUR); - return { - aid: video.aid, - bvid: video.bvid, - description: video.desc, - uid: video.owner.mid, - tags: null, - title: video.title, - published_at: published_at, - } as AllDataType; - }); + return data.data.archives; } catch (error) { logger.error(error as Error, "net", "getLatestVideos"); return null; diff --git a/lib/task/insertLatestVideo.ts b/lib/task/insertLatestVideo.ts index af932fb..e6b750b 100644 --- a/lib/task/insertLatestVideo.ts +++ b/lib/task/insertLatestVideo.ts @@ -9,7 +9,6 @@ import logger from "lib/log/logger.ts"; export async function insertLatestVideos( client: Client, pageSize: number = 10, - sleepRate: number = 250, intervalRate: number = 4000, ): Promise { const latestVideoTimestamp = await getLatestVideoTimestampFromAllData(client); @@ -27,7 +26,7 @@ export async function insertLatestVideos( for (const video of videoIndex) { const videoExists = await videoExistsInAllData(client, video.aid); if (!videoExists) { - insertIntoAllData(client, video); + await insertIntoAllData(client, video); } } return 0; @@ -37,7 +36,7 @@ export async function insertLatestVideos( const insertedVideos = new Set(); while (true) { try { - const videos = await getLatestVideos(page, pageSize, sleepRate); + const videos = await getLatestVideos(page, pageSize); if (videos == null) { failCount++; if (failCount > 5) { @@ -53,7 +52,7 @@ export async function insertLatestVideos( for (const video of videos) { const videoExists = await videoExistsInAllData(client, video.aid); if (!videoExists) { - insertIntoAllData(client, video); + await insertIntoAllData(client, video); insertedVideos.add(video.aid); } } @@ -68,7 +67,7 @@ export async function insertLatestVideos( if (failCount > 5) { return null; } - continue; + } finally { await sleep(Math.random() * intervalRate + failCount * 3 * SECOND + SECOND); }