alikia2x/cvsa
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add: more logging for filter's workers

Browse Source
This commit is contained in:
alikia2x (寒寒) 2025-02-22 21:43:09 +08:00
parent bdbd2ac748
commit f70401846a
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
3 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/db/allData.ts
View File

@ -77,7 +77,7 @@ export async function getNullVideoTagsList(client: Client) {
); );
} }
export async function getUnlabeledVideos(client: Client) { export async function getUnlabelledVideos(client: Client) {
const queryResult = await client.queryObject<{ aid: number }>( const queryResult = await client.queryObject<{ aid: number }>(
`SELECT a.aid FROM all_data a LEFT JOIN labelling_result l ON a.aid = l.aid WHERE l.aid IS NULL`, `SELECT a.aid FROM all_data a LEFT JOIN labelling_result l ON a.aid = l.aid WHERE l.aid IS NULL`,
); );

1
lib/ml/filter_inference.ts
View File

@ -28,6 +28,7 @@ export async function initializeModels() {
sessionClassifier = classifierSession; sessionClassifier = classifierSession;
sessionEmbedding = embeddingSession; sessionEmbedding = embeddingSession;
logger.log("Filter models initialized", "ml");
} catch (error) { } catch (error) {
const e = new WorkerError(error as Error, "ml", "fn:initializeModels"); const e = new WorkerError(error as Error, "ml", "fn:initializeModels");
throw e; throw e;

5
lib/mq/exec/classifyVideo.ts
View File

@ -1,6 +1,6 @@
import { Job } from "bullmq"; import { Job } from "bullmq";
import { db } from "lib/db/init.ts"; import { db } from "lib/db/init.ts";
import { getUnlabeledVideos, getVideoInfoFromAllData, insertVideoLabel} from "lib/db/allData.ts"; import { getUnlabelledVideos, getVideoInfoFromAllData, insertVideoLabel} from "lib/db/allData.ts";
import { classifyVideo, initializeModels } from "lib/ml/filter_inference.ts"; import { classifyVideo, initializeModels } from "lib/ml/filter_inference.ts";
import { ClassifyVideoQueue } from "lib/mq/index.ts"; import { ClassifyVideoQueue } from "lib/mq/index.ts";
import logger from "lib/log/logger.ts"; import logger from "lib/log/logger.ts";
@ -35,7 +35,8 @@ export const classifyVideoWorker = async (job: Job) => {
export const classifyVideosWorker = async () => { export const classifyVideosWorker = async () => {
await initializeModels(); await initializeModels();
const client = await db.connect(); const client = await db.connect();
const videos = await getUnlabeledVideos(client); const videos = await getUnlabelledVideos(client);
logger.log(`Found ${videos.length} unlabelled videos`)
client.release(); client.release();
let i = 0; let i = 0;
for (const aid of videos) { for (const aid of videos) {