78 lines
2.5 KiB
TypeScript
78 lines
2.5 KiB
TypeScript
import { Client } from "https://deno.land/x/postgres@v0.19.3/mod.ts";
|
|
import { getLatestVideos } from "lib/net/getLatestVideos.ts";
|
|
import { getLatestVideoTimestampFromAllData, insertIntoAllData, videoExistsInAllData } from "lib/db/allData.ts";
|
|
import { sleep } from "lib/utils/sleep.ts";
|
|
import { getVideoPositionInNewList } from "lib/net/bisectVideoStartFrom.ts";
|
|
import { SECOND } from "$std/datetime/constants.ts";
|
|
import logger from "lib/log/logger.ts";
|
|
|
|
export async function insertLatestVideos(
|
|
client: Client,
|
|
pageSize: number = 10,
|
|
sleepRate: number = 250,
|
|
intervalRate: number = 4000,
|
|
): Promise<number | null> {
|
|
const latestVideoTimestamp = await getLatestVideoTimestampFromAllData(client);
|
|
if (latestVideoTimestamp == null) {
|
|
logger.error("Cannot get latest video timestamp from current database.", "net", "fn:insertLatestVideos()");
|
|
return null
|
|
}
|
|
logger.log(`Latest video in the database: ${new Date(latestVideoTimestamp).toISOString()}`, "net", "fn:insertLatestVideos()")
|
|
const videoIndex = await getVideoPositionInNewList(latestVideoTimestamp);
|
|
if (videoIndex == null) {
|
|
logger.error("Cannot locate the video through bisect.", "net", "fn:insertLatestVideos()");
|
|
return null
|
|
}
|
|
if (typeof videoIndex == "object") {
|
|
for (const video of videoIndex) {
|
|
const videoExists = await videoExistsInAllData(client, video.aid);
|
|
if (!videoExists) {
|
|
insertIntoAllData(client, video);
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
let page = Math.floor(videoIndex / pageSize) + 1;
|
|
let failCount = 0;
|
|
const insertedVideos = new Set();
|
|
while (true) {
|
|
try {
|
|
const videos = await getLatestVideos(page, pageSize, sleepRate);
|
|
if (videos == null) {
|
|
failCount++;
|
|
if (failCount > 5) {
|
|
return null;
|
|
}
|
|
continue;
|
|
}
|
|
failCount = 0;
|
|
if (videos.length == 0) {
|
|
logger.verbose("No more videos found", "net", "fn:insertLatestVideos()");
|
|
break;
|
|
}
|
|
for (const video of videos) {
|
|
const videoExists = await videoExistsInAllData(client, video.aid);
|
|
if (!videoExists) {
|
|
insertIntoAllData(client, video);
|
|
insertedVideos.add(video.aid);
|
|
}
|
|
}
|
|
logger.log(`Page ${page} crawled, total: ${insertedVideos.size} videos.`, "net", "fn:insertLatestVideos()");
|
|
page--;
|
|
if (page < 1) {
|
|
return 0;
|
|
}
|
|
} catch (error) {
|
|
logger.error(error as Error, "net", "fn:insertLatestVideos()");
|
|
failCount++;
|
|
if (failCount > 5) {
|
|
return null;
|
|
}
|
|
continue;
|
|
} finally {
|
|
await sleep(Math.random() * intervalRate + failCount * 3 * SECOND + SECOND);
|
|
}
|
|
}
|
|
return 0;
|
|
}
|