alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update: insert duration as metadata

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-02-14 03:06:39 +08:00
parent bfc622bfda
commit 4b48357ab6
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
4 changed files with 33 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
lib/db/allData.ts
View File

@ -1,20 +1,26 @@
import { Client, Transaction } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
import { AllDataType } from "lib/db/schema.d.ts";
import logger from "lib/log/logger.ts";
import { parseTimestampFromPsql } from "lib/utils/formatTimestampToPostgre.ts";
import { formatTimestampToPsql, parseTimestampFromPsql } from "lib/utils/formatTimestampToPostgre.ts";
import { VideoListVideo } from "lib/net/bilibili.d.ts";
import { HOUR, SECOND } from "$std/datetime/constants.ts";
export async function videoExistsInAllData(client: Client, aid: number) {
return await client.queryObject<{ exists: boolean }>(`SELECT EXISTS(SELECT 1 FROM all_data WHERE aid = $1)`, [aid])
.then((result) => result.rows[0].exists);
}
export async function insertIntoAllData(client: Client, data: AllDataType) {
export async function biliUserExists(client: Client, uid: number) {
return await client.queryObject<{ exists: boolean }>(`SELECT EXISTS(SELECT 1 FROM bili_user WHERE uid = $1)`, [uid])
.then((result) => result.rows[0].exists);
}
export async function insertIntoAllData(client: Client, data: VideoListVideo) {
logger.log(`inserted ${data.aid}`, "db-all_data");
return await client.queryObject(
`INSERT INTO all_data (aid, bvid, description, uid, tags, title, published_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
await client.queryObject(
`INSERT INTO all_data (aid, bvid, description, uid, tags, title, published_at, duration)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (aid) DO NOTHING`,
[data.aid, data.bvid, data.description, data.uid, data.tags, data.title, data.published_at],
[data.aid, data.bvid, data.desc, data.owner.mid, null, data.title, formatTimestampToPsql(data.pubdate * SECOND + 8 * HOUR), data.duration],
);
}

24
lib/net/bisectVideoStartFrom.ts
View File

@ -1,7 +1,8 @@
import { getLatestVideos } from "lib/net/getLatestVideos.ts";
import { AllDataType } from "lib/db/schema.d.ts";
import { HOUR, SECOND } from "$std/datetime/constants.ts";
import { VideoListVideo } from "lib/net/bilibili.d.ts";
export async function getVideoPositionInNewList(timestamp: number): Promise<number | null | AllDataType[]> {
export async function getVideoPositionInNewList(timestamp: number): Promise<number | null | VideoListVideo[]> {
const virtualPageSize = 50;
let lowPage = 1;
@ -10,16 +11,15 @@ export async function getVideoPositionInNewList(timestamp: number): Promise<numb
while (true) {
const ps = highPage < 2 ? 50 : 1
const pn = highPage < 2 ? 1 : highPage * virtualPageSize;
const fetchTags = highPage < 2 ? true : false;
const videos = await getLatestVideos(pn, ps, 250, fetchTags);
const videos = await getLatestVideos(pn, ps);
if (!videos || videos.length === 0) {
break;
}
const lastVideo = videos[videos.length - 1];
if (!lastVideo || !lastVideo.published_at) {
if (!lastVideo || !lastVideo.pubdate) {
break;
}
const lastTime = Date.parse(lastVideo.published_at);
const lastTime = lastVideo.pubdate * SECOND + 8 * HOUR;
if (lastTime <= timestamp && highPage == 1) {
return videos;
}
@ -41,7 +41,7 @@ export async function getVideoPositionInNewList(timestamp: number): Promise<numb
let hi = highPage;
while (lo <= hi) {
const mid = Math.floor((lo + hi) / 2);
const videos = await getLatestVideos(mid * virtualPageSize, 1, 250, false);
const videos = await getLatestVideos(mid * virtualPageSize, 1);
if (!videos) {
return null;
}
@ -50,11 +50,11 @@ export async function getVideoPositionInNewList(timestamp: number): Promise<numb
continue;
}
const lastVideo = videos[videos.length - 1];
if (!lastVideo || !lastVideo.published_at) {
if (!lastVideo || !lastVideo.pubdate) {
hi = mid - 1;
continue;
}
const lastTime = Date.parse(lastVideo.published_at);
const lastTime = lastVideo.pubdate * SECOND + 8 * HOUR;
if (lastTime > timestamp) {
lo = mid + 1;
} else {
@ -63,15 +63,15 @@ export async function getVideoPositionInNewList(timestamp: number): Promise<numb
}
}
const boundaryVideos = await getLatestVideos(boundaryPage, virtualPageSize, 250, false);
const boundaryVideos = await getLatestVideos(boundaryPage, virtualPageSize);
let indexInPage = 0;
if (boundaryVideos && boundaryVideos.length > 0) {
for (let i = 0; i < boundaryVideos.length; i++) {
const video = boundaryVideos[i];
if (!video.published_at) {
if (!video.pubdate) {
continue;
}
const videoTime = Date.parse(video.published_at);
const videoTime = video.pubdate * SECOND + 8 * HOUR;
if (videoTime > timestamp) {
indexInPage++;
} else {

24
lib/net/getLatestVideos.ts
View File

@ -1,15 +1,10 @@
import { VideoListResponse } from "lib/net/bilibili.d.ts";
import { formatTimestampToPsql as formatPublishedAt } from "lib/utils/formatTimestampToPostgre.ts";
import { AllDataType } from "lib/db/schema.d.ts";
import { VideoListResponse, VideoListVideo } from "lib/net/bilibili.d.ts";
import logger from "lib/log/logger.ts";
import { HOUR, SECOND } from "$std/datetime/constants.ts";
export async function getLatestVideos(
page: number = 1,
pageSize: number = 10,
sleepRate: number = 250,
fetchTags: boolean = true,
): Promise<AllDataType[] | null> {
pageSize: number = 10
): Promise<VideoListVideo[] | null> {
try {
const response = await fetch(
`https://api.bilibili.com/x/web-interface/newlist?rid=30&ps=${pageSize}&pn=${page}`,
@ -26,18 +21,7 @@ export async function getLatestVideos(
return [];
}
return data.data.archives.map((video) => {
const published_at = formatPublishedAt(video.pubdate * SECOND + 8 * HOUR);
return {
aid: video.aid,
bvid: video.bvid,
description: video.desc,
uid: video.owner.mid,
tags: null,
title: video.title,
published_at: published_at,
} as AllDataType;
});
return data.data.archives;
} catch (error) {
logger.error(error as Error, "net", "getLatestVideos");
return null;

9
lib/task/insertLatestVideo.ts
View File

@ -9,7 +9,6 @@ import logger from "lib/log/logger.ts";
export async function insertLatestVideos(
client: Client,
pageSize: number = 10,
sleepRate: number = 250,
intervalRate: number = 4000,
): Promise<number | null> {
const latestVideoTimestamp = await getLatestVideoTimestampFromAllData(client);
@ -27,7 +26,7 @@ export async function insertLatestVideos(
for (const video of videoIndex) {
const videoExists = await videoExistsInAllData(client, video.aid);
if (!videoExists) {
insertIntoAllData(client, video);
await insertIntoAllData(client, video);
}
}
return 0;
@ -37,7 +36,7 @@ export async function insertLatestVideos(
const insertedVideos = new Set();
while (true) {
try {
const videos = await getLatestVideos(page, pageSize, sleepRate);
const videos = await getLatestVideos(page, pageSize);
if (videos == null) {
failCount++;
if (failCount > 5) {
@ -53,7 +52,7 @@ export async function insertLatestVideos(
for (const video of videos) {
const videoExists = await videoExistsInAllData(client, video.aid);
if (!videoExists) {
insertIntoAllData(client, video);
await insertIntoAllData(client, video);
insertedVideos.add(video.aid);
}
}
@ -68,7 +67,7 @@ export async function insertLatestVideos(
if (failCount > 5) {
return null;
}
continue;
} finally {
await sleep(Math.random() * intervalRate + failCount * 3 * SECOND + SECOND);
}