update: labeling for filter

data/filter/labeled_data.jsonl

@@ -915,4 +915,19 @@
 {"aid": 685634071, "title": "【AI歌手】腐草为萤【Cover】", "description": "原唱：银临\n引擎：歌叽歌叽\nPV借用：av2700587\nPV：星の祈\n绘：otakucake阿饼（P站：pixiv.me/otakucake）", "tags": ["AI", "歌叽歌叽"], "author_info": "一支粉笔w: 人生难得一知己，千古知音最难求。 | 网易云：粉笔 微博：http://weibo.com/Chalk10", "label": 2}
 {"aid": 3372898, "title": "【言和英文】Counting Stars", "description": "自制 终于投了。1.封面仍然灵魂p图。2.这玩意儿9月底就在做了，说好10月投，pv还解决不了。说好11月投，但又因为曲子本身又拖了。上个星期想投，结果，电脑坏掉加发烧。。。。。所以一路拖到今天才投。算是放下心中的一块石头了吧。3.这次画了很多参数，感觉有提升，不知道朱军是怎么看的。5.counting stars是我最最最喜欢的一首歌，已经循环一年。6.1R赶紧出4专！！！", "tags": ["VOCALOID", "COUNTING STARS", "言和", "ONEREPUBLIC"], "author_info": "JackChenZz: 随缘更新", "label": 0}
 {"aid": 589575, "title": "【洛天依】约定", "description": "绫：天依，不管几年之后你能否记得我，我会一直恋你，爱你，铭心记住你！ QAQ 被小汐撺掇调的，咱非常不擅长这拐来拐去的长调式滴说，不过这次应该算不上渣吧？（好吧- -无视这句）。翻的周慧的《约定》，谱子依旧自己来（其实只是自己重做MIDI了，网上的MIDI做觉得不好用，调子太高），3分43秒的字幕是UP无聊的XXX。PV用慢放的《春来发几只》弄的，剪掉了两组燕子镜头~ ~赶脚两个曲子的BPM同步了", "tags": ["乐正绫", "洛天依", "洛天依翻唱曲", "VOCALOID", "良调教", "约定", "周慧"], "author_info": "星璇の天空: Dr.冥月星璇 成就：不会调教的调教师（U/V/SV/C)，专业咸鱼，不会科研的副研究员，不会编程的系统架构师，不会AI的无人机设计师...", "label": 2}
-{"aid": 411693256, "title": "【言和V5】独二无一【VOCALOID Cover】", "description": "人声本家：https://www.bilibili.com/video/BV1oJ411u7Xq?t=222.3\nVC本家：https://www.bilibili.com/video/BV12J411u7Aa?t=2.8", "tags": ["Vsinger创作激励计划", "VOCALOID CHINA", "VOCALOID", "言和", "虚拟歌手", "独二无一", "Vsinger创作激励计划第六期"], "author_info": "远宇V: 言和vsinger\njiafei同人扣扣群:121249682", "label": 0}
+{"aid": 411693256, "title": "【言和V5】独二无一【VOCALOID Cover】", "description": "人声本家：https://www.bilibili.com/video/BV1oJ411u7Xq?t=222.3\nVC本家：https://www.bilibili.com/video/BV12J411u7Aa?t=2.8", "tags": ["Vsinger创作激励计划", "VOCALOID CHINA", "VOCALOID", "言和", "虚拟歌手", "独二无一", "Vsinger创作激励计划第六期"], "author_info": "远宇V: 言和vsinger\njiafei同人扣扣群:121249682", "label": 0}{"aid": 113244177962166, "title": "【スーサイド（Suicide！）×  头痛欲绝】头痛到自 杀了呢", "description": "-", "tags": ["重音テト", "頭ン痛", "头痛欲绝", "スーサイド！", "Suicide！", "teto"], "author_info": "硕果-累累: 该up主视频可能有长时间断更（不可催更因为要上课（乐））不足之处欢迎提出意见", "label": 0}
+{"aid": 791366733, "title": "menfnfn", "description": "-", "tags": ["让虚拟歌手为你唱", "虚拟之声创作计划·2023第四期"], "author_info": "bili_70413135045: ", "label": 0}
+{"aid": 233901881, "title": "《父亲的草原母亲的河》-降D调-钢琴伴奏-正谱伴奏-动态谱", "description": "声乐作品钢琴正谱伴奏视频，音频纯钢琴录制，视频动态走带条手工调校，让声乐教学更高效，练歌更方便。\n学生学歌神器，老师教学助手！\n说明：降调伴奏仅为音频降调，曲谱仍为原调，需要原调伴奏可进入个人空间搜索……", "tags": ["虚拟歌手分享官", "教学", "音乐", "正谱伴奏", "和伴奏", "虚拟之声创作计划·2023第三期", "钢琴伴奏", "培训", "歌曲伴奏", "声乐伴奏", "必剪创作"], "author_info": "音乐视频玩家: 来一首说唱就唱的歌曲吧，音乐、歌词都准备好了！              \n\n#…分享各类艺术歌曲及伴奏…#", "label": 0}
+{"aid": 91991603, "title": "【洛天依】洛天依 - 盛开季节", "description": "作品类型: 原创曲\n调教: 阿宽\n作词: 阿宽\n作曲: 阿宽\n编曲/混音: 阿宽\n简介补充: 创作于10.03.10，新编于20.02.26。。。我现在是多无聊。。。。_(:з」∠)_", "tags": ["VOCALOID", "VOCALOIDCHINA", "原创歌曲", "原创曲", "自制", "洛天依原创曲", "洛天依"], "author_info": "arqalm: 我的网易音乐人主页https://music.163.com/#/artist?id=12126188", "label": 1}
+{"aid": 1406015081, "title": "「 AI 孙燕姿 」《请你一定要记得我好吗》cover 房东的猫", "description": "免责声明:\n本作品仅作为娱乐目的发布，可能造成的后果与使用的音声转换项目的作者、贡献者无关，\n本视频使用AI音色替换技术(Sovits4.0)，视频中演唱人声并非孙燕姿本人。\n如有侵权告知删除\n原唱：请你一定要记得我好吗-房东的猫\nAI声源：孙燕姿", "tags": ["AI", "孙燕姿", "COVER", "翻唱", "房东的猫"], "author_info": "无语猫猫没有鱼: 要好好吃饭哦", "label": 2}
+{"aid": 236266867, "title": "在一起真好｜一緒がよかった / 重音テトSV【SynthV オリジナル】", "description": "https://youtu.be/8H8P_w4OAmo\nUP：Haqua - はふれつ\n一緒がよかった / 重音テトSV【SynthV オリジナル】\n11月22日\n2023年\n関係ないけどT1優勝おめでとう；；", "tags": ["携手虚拟歌手征服世界吧！", "虚拟歌手", "重音テト", "SynthV"], "author_info": "音街ウナ: AI觉醒！切蒲英！\n          ——2021.2.9\n是音街鳗切蒲英双推人，也看冷门歌姬和VUP\n有想让我搬运的油管视频欢迎来推荐哦", "label": 0}
+{"aid": 212218392, "title": "【鏡音レン】嫉妬[short ver]【yoh】", "description": "sm40137018\n投稿日時 2022/03/06 18:00\n嫉妬\n\nvo : 鏡音レン\n\nillust : 조둔이 \nhttps://twitter.com/JODOONI\n\nmusic & video : yoh\nhttps://twitter.com/yoh_hidamari\n\noff vo→https://www.dropbox.com/s/wbvl5gew4oxb5u4/%E5%AB%89%E5%A6%AC%20off.mp3?dl=0", "tags": ["VOCALOID", "鏡音レン", "yoh", "嫉妬"], "author_info": "路过的某咸鱼: ", "label": 0}
+{"aid": 18174891, "title": "【乐正龙牙&洛天依】凉凉（考试版） 献给挂科或者即将挂科的小伙伴们", "description": "喜欢的话就推荐、投币、收藏、订阅素质4连吧~~\r\n翻调曲\r\npv:DS绝对小贱\r\n重作词:@凸凸你034\r\n英文翻译:@凸凸你034\r\n调教:DS绝对小贱\r\n新浪微博：http://weibo.com/dsxiaojian2000\n\r\n我优酷空间地址：http://i.youku.com/i/UMTIwNjYwNTQwOA==", "tags": ["翻译大家", "洛天依", "翻唱", "乐正龙牙", "VOCALOID中文曲", "VOCALOID", "合唱", "开口跪", "凉凉", "重作词"], "author_info": "DS绝对小贱: 鬼畜区的宝藏up 中国版《我的世界》DoMCer服务器 Admin", "label": 2}
+{"aid": 1505358010, "title": "【洛天依V4J】熱異常【VOCALOIDカバー】", "description": "■重新调了一下…上次听的怪怪的是因为加了自然音效，这次对整体都做了一定的修正和更改\n\n⚠☞但是声音还是有点小，耳机党请增大音量🔊\n-----------------------------------\n■本家：https://youtu.be/b2NTglk9tvI?si=XHhYTbeMtgNWNYG6\n\n☞参考的视频以及ust（转vsqx）：\n■https://bowlroll.net/file/286986\n■https://b23.tv/9uN5ER1\n-----------------------------------\n■调音（混音）：我\n■编辑器：VOCALOID6", "tags": ["VOCALOID", "V4", "洛天依", "日语翻唱", "熱異常", "いよわP", "VOCALOIDカバー"], "author_info": "伊従蒼: ", "label": 0}
+{"aid": 383098671, "title": "【雀河＆长歌】《知否知否》（荼鸢：什么时候轮到我？ 我：下次一定！）", "description": "原唱：胡夏/郁可唯\n翻唱：雀河/长歌\n调教：本人\n本来想用荼鸢，可是长歌实在太适合了……", "tags": ["让虚拟歌手为你唱", "长歌", "ACE虚拟歌姬", "雀河", "中文VOCALOID", "知否知否"], "author_info": "松下_下上上左右AB: Jam project YYDS", "label": 2}
+{"aid": 4983329, "title": "【洛天依原创】九歌·东皇太一", "description": "自制 1.灵感来了无法阻挡，灵感欲去转瞬即逝啊朋友们。→_→ 2.对，等有空了就不再做这么敷衍的pv了→_→。", "tags": ["中国风", "古风", "VOCALOID", "VOCALOID中文曲", "洛天依", "屈原"], "author_info": "十六air: 你所热爱的，就是说的道理。", "label": 1}
+{"aid": 580592330, "title": "【言和v3】不安灵魂收容所（short ver.）【60分调声】", "description": "不安言和收容所！\n翻唱：言和\n调：团子\n混：ngf\nvsqx：719\n去年（怎么已经是去年了）60分调声的存稿，混个新年曲了\n很粗糙的调以及跨语种，并且为了能按时调完还把和声轨删了…………随便听听得了\n\n\n\n原曲：BV1Nu411t7Pj\n「不安灵魂收容所」\n词曲、制作人：ChiliChill\n人声录音室：RisingToneStudio\n曲绘：kurattes\nPV制作：缘聚世代", "tags": ["Vsinger创作激励计划", "VOCALOID CHINA", "言和", "COVER", "ChiliChill", "中文VOCALOID", "Vsinger创作激励计划第六期"], "author_info": "团子_P: 不在别找不知道", "label": 2}
+{"aid": 113725465957955, "title": "【鏡音リン17th】花海【中日双语翻唱】", "description": "原唱：周杰伦\n翻唱：鏡音リン act1 （日语部分）\n           鏡音リン power/warm （中文部分）\n背景：Project SEKAI 镜音铃17周年生贺卡面\n           自摄\nリンちゃん生日快乐！\n三次元繁忙，赶制的生贺翻唱，混音依旧一坨。\nC社！！！！铃的新声库什么时候出！！！", "tags": ["花海", "COVER", "镜音铃", "鏡音リン", "翻唱", "VOCALOID"], "author_info": "纯情好少年: 词不达意，表达模糊，语文不及格工科狗", "label": 2}
+{"aid": 66426751, "title": "【洛天依V4J】首班车和卡夫卡【COVER】", "description": "Music·Lyrics：ナブナsm27057005/av2840917\nVocal: 洛天依\nManipulation：多元P\nvsqx：ヤ-クザPav38709954\nLast Work：av64325315\n\n小伙伴说了，只要想吃冰，无论何时都是夏天xxxxxx", "tags": ["VOCALOID", "ナブナ", "洛天依日语", "洛天依", "始発とカフカ", "首班车和卡夫卡", "n-buna"], "author_info": "多元P: 日语依不稳定产粮ing", "label": 0}
+{"aid": 4121667, "title": "【洛天依】出狱新单《哈密瓜》", "description": "自制 视频：乔木 词曲编混唱：舟扒皮\r\n\r\n终于放出来了，哈密瓜好好吃哦。", "tags": ["原创曲", "洛天依", "VOCALOID", "舟扒皮"], "author_info": "舟皮: ", "label": 1}
+{"aid": 58775982, "title": "【洛天依】祖龙吟", "description": "以下是原曲staff\n总策划：木宁木蒙\n企划运营：塔库\n监制：落落无尘、卿雅、小仙、顾雪柔\n作曲：陈亦洺 | 作词：玄天 | 编曲：Mzf小慕\n演唱：星尘 | 调教：花儿不哭 | 混音：Mr.曾经 | 笛子：囚牛\n视频：小约酱【麻薯映像】| 曲绘：白鄔東\n朕之功过，当世无可评说者，百代之后自有论断。\nvsqx：桂月 & 星葵 & 熊\n原曲：av42348447\n天依生日快乐！！！\n我永远喜欢洛天依！！！", "tags": ["VOCALOIDCHINA", "洛天依", "中国风", "古风", "2019洛天依生日会"], "author_info": "洛水清平: 取次花丛懒回顾，半缘修道半缘君。 ", "label": 2}

data/filter/real_test.jsonl

@@ -101,3 +101,17 @@
 {"aid": 56016654, "title": "【重音テト】あっけない【./わさび】", "tags": ["公主殿下"], "author_info": "neneneneneneneko: ", "description": "sm35282409\nボカロ23曲目。ウタウ。\n\nボカロ曲\nmylist/57784222\nインスト曲\nmylist/53204966\ntwitter\nhttps://twitter.com/wasa_giga", "model": 0, "human": 0}
 {"aid": 579224517, "title": "∧…在", "tags": ["让虚拟歌手为你唱", "虚拟之声创作计划·2023第四期"], "author_info": "时序夜: ", "description": "-", "model": 0, "human": 0}
 {"aid": 1351556126, "title": "フルグア GalGame姬恋~缚羽的欠片OP -- 心之芽 vocal by 初音未来", "tags": ["OP", "初音未来", "ACG音乐", "主题曲", "姬恋缚羽的欠片"], "author_info": "Chinbam_: 离开中...", "description": "GalGame姬恋~缚羽的欠片OP\n玩GAlGAME发现主题曲挺好听看站内貌似没有就投一下自己听 hhh\n作词·作曲·编曲：フルグア\n视频：灼弦&Afezeria&沛之暗影\n能不能过审也不知道 hh\n---侵删", "model": 0, "human": 0}
+{"aid": 68443207, "title": "レジデュアム _ 鏡音リン - ニコニコ動画", "tags": ["镜音RIN"], "author_info": "neneneneneneneko: ", "description": "sm18807181", "model": 0, "human": 0}
+{"aid": 1847414, "title": "【初音ミク】 光谱 【Falconnect】", "tags": ["VOCALOID", "初投稿", "初音MIKU", "字幕付", "ハヤブサ", "FALCONNECT", "キミスペクトル"], "author_info": "糜酱: 〓如视频侵权请私信我删〓\r\n\r\n    视频↙可调原画。  下载自助  http://www.bilibilijj.com/oldindex\r\n\r\n「何も知らずに勝手に誤解されるのはごめんだ。」\r\n", "description": "sm25250566 标题求翻译； ；\r\n【Falconnect】\r\n■music :ハヤブサ\r\n■illust　:桜木蓮\r\n■movie :まつらい\r\nカラオケはこちら。https://www.dropbox.com/home/Vocal%20off%20Track", "model": 0, "human": 0}
+{"aid": 430054078, "title": "【オリジナル曲】孤独回廊 ／ イナガシ feat.初音ミク【ニコニコ転載】", "tags": ["转载", "日语", "MIKU", "NICONICO", "初音MIKU", "初音", "公主殿下", "初音未來"], "author_info": "定電流: ^%t&#a", "description": "https://www.nicovideo.jp/watch/sm40996727", "model": 0, "human": 0}
+{"aid": 977788326, "title": "【东方栀子Era】最美的期待【VocalSharp COVER】", "tags": ["东方栀子", "VocalSharp", "东方栀子十周年投稿祭"], "author_info": "口古口古口古郁: ", "description": "原版：周笔畅《最美的期待》\n翻唱：东方栀子Era\n\nVS巨好用！摸鱼飞快！\n早上起来突然想摸，要不是出去走亲戚中午就能投了，焯", "model": 2, "human": 2}
+{"aid": 295971472, "title": "一梦千宵【乐正绫cover】", "tags": ["必剪创作", "2022Vsinger创作激励计划第一期"], "author_info": "ViVi141: ", "description": "较上个视频优化了一下混音", "model": 2, "human": 2}
+{"aid": 630517376, "title": "【ニコカラ】パラジクロロベンゼン（对二氯苯）(off vocal)【鏡音レン】", "tags": ["VOCALOID", "对二氯苯", "镜音连", "パラジクロロベンゼン", "オワタP", "やさ", "完蛋P", "nicokara"], "author_info": "诳原糸: なぜ日は傾くのか", "description": "sm8346614\n自身の持ち込み用に作成させて頂きました。\n\nお借りした原曲様→sm8269164\n作ってみたリスト→mylist/9495351\n\n【10/6追記】\non vocal版作成させて頂きました→sm8438926\n——やさ", "model": 0, "human": 0}
+{"aid": 34280265, "title": "夜景", "tags": ["デフォ子", "鷹々音タカユキ"], "author_info": "neneneneneneneko: ", "description": "sm34052035\n参加音源　デフォ子\n「ナカノは４番」さんの感じ出したかったけど微妙かなー \n今回　歌詞表示してません\nあと　イントロ長いです\n\nツイッター@pz0Z2IxidBJbNyk", "model": 0, "human": 0}
+{"aid": 1353871111, "title": "可不也入飞门了 《最佳顺友》【CeVIO AI 粤语cover】", "tags": ["搞笑", "英雄联盟", "飞门", "CeVIO AI", "飞马", "张顺飞", "可不", "原神"], "author_info": "事隠しのfubuki: ", "description": "原作者 ：原家门-杰顺克  本家：BV1sq421P7bC\n感谢老哥授权二创，可惜可不把老哥的奥德腮皮肤遮住了\n\n混/调：我\n\n可不图作者推特：https://x.com/noizman_sympho\n\nmocha pro 总是跟踪失败，用剪映摆了，这几天脑子里全是这歌，太顶级了", "model": 2, "human": 2}
+{"aid": 248109170, "title": "（传说之下游戏）Treo的尘埃的信任一段开头演示官方重置 Dusttrust phase 1 thingie", "tags": ["BGM", "AU", "传说之下", "UT", "UNDERTALEAU"], "author_info": "三尘-深渊: 画渣，无小号，只在B站发视频和动态", "description": "该视频作者 Treo 3340位订阅者\n原视频链接https://m.youtube.com/watch?v=1F3EpHU1vnI", "model": 0, "human": 0}
+{"aid": 113233474097330, "title": "【永夜Minus】Bleeding Love", "tags": ["让虚拟歌手为你唱", "虚拟歌姬", "虚拟歌手", "音乐", "VOCALOID", "Synthesizer V", "永夜Minus", "虚拟之声创作计划·2024第三期", "翻调", "Leona Lewis", "Bleeding Love"], "author_info": "五等分的monop: 菜菜调校", "description": "原曲：Bleeding Love\n原唱：Leona Lewis\n-----------------------------------------------------\n-----------------------------------------------------\n调校：monop\n混音：monop", "model": 2, "human": 0}
+{"aid": 115025, "title": "【初音ミク】Re:birth ☆实长: 3:19", "description": "sm15005742 ☆实际长度: 3:19 ஐMusic&Lyrics：キョーヘイ ஐIllust：LiLLa 久し振りに病んだ曲になってますｗ.こんな曲は作ってて楽しいですねｗｗ Re:birth?重生?似乎可以这样理解吧?ˋ ( ° ▽、° )", "tags": ["初音MIKU", "VOCALOID", "キョーヘイ", "VOCALOID新曲リンク"], "author_info": "雪暴公主: Forever Alone", "url": "https://www.bilibili.com/video/BV1Qx411c7GE", "model_label": 0, "user_label": null, "human": 0}
+{"aid": 112657831036502, "title": "冰火歌会 2", "description": "-", "tags": ["冰火歌会", "必剪创作", "虚拟 UP 主", "虚拟最强音·2024第三期"], "author_info": "LV驴子: ", "url": "https://www.bilibili.com/video/BV1AegbezEC8", "model_label": 0, "user_label": null, "human": 0}
+{"aid": 732450889, "title": "【洛天依AI】四季予你【ACE Studio COVER】", "description": "四季予你\n作词：烟十八\n作曲：郑冰冰\n编曲：刘圣华（华子）\n原唱：程响\n翻唱：洛天依\n调教：夏龙杰lndulge\n求大佬评论指导", "tags": ["让虚拟歌手为你唱", "VOCALOID CHINA", "COVER", "洛天依", "虚拟歌手创意音乐大赏"], "author_info": "夏龙杰lndulge: 又名怠惰p，高中生，约稿B站私信可约调教和混音（混音最好别约）不求关注，但求三连加评论 （能关注一下也可以）", "url": "https://www.bilibili.com/video/BV1VD4y1x7c1", "model_label": 2, "user_label": null, "human": 2}
+{"aid": 310042157, "title": "【心华翻唱】祖龙吟【忘川风华录】", "description": "总策划：木宁木蒙\n企划运营：塔库\n监制：落落无尘、卿雅、小仙、顾雪柔\n作曲：陈亦洺 | 作词：玄天 | 编曲：Mzf小慕\n演唱：星尘 | 调教：花儿不哭 | 混音：Mr.曾经 | 笛子：囚牛\n视频：小约酱【麻薯映像】| 曲绘：白鄔東\n朕之功过，当世无可评说者，百代之后自有论断。\n\n原作指路;BV11b411k7ym\n\n感谢忘川的公开二创素材，调了一个半月了，给孩子一个三连吧求求了。。。", "tags": ["虚拟之声创作计划", "COVER", "心华", "翻唱", "VOCALOID", "VOCALOID CHINA", "嬴政", "忘川风华录", "祖龙吟", "虚拟之声创作计划·2023第一期"], "author_info": "幽影丶鎺: 一只鸽子精，约稿事宜请私信哦~咕咕咕", "url": "https://www.bilibili.com/video/BV1UA411172s", "model_label": 2, "user_label": null, "human": 2}

filter/db_utils.py

@@ -0,0 +1,33 @@
+import sqlite3
+import json
+
+def fetch_entry_data(db_path, aid):
+    """
+    根据aid从数据库中加载data
+    """
+    conn = sqlite3.connect(db_path)
+    cursor = conn.cursor()
+    cursor.execute("SELECT data FROM bili_info_crawl WHERE aid = ?", (aid,))
+    fet = cursor.fetchone()
+    if fet:
+        data = fet[0]
+    else:
+        data = None
+    conn.close()
+    return data
+
+def parse_entry_data(data):
+    """
+    解析JSON数据，提取视频标题、简介、标签、作者简介
+    """
+    try:
+        obj = json.loads(data)
+        title = obj["View"]["title"]
+        description = obj["View"]["desc"]
+        tags = [tag["tag_name"] for tag in obj["Tags"] if tag["tag_type"] in ["old_channel", "topic"]]
+        author_info = obj["Card"]["card"]["name"] + ": " + obj["Card"]["card"]["sign"]
+        url = "https://www.bilibili.com/video/" + obj["View"]["bvid"]
+        return title, description, tags, author_info, url
+    except (TypeError, json.JSONDecodeError) as e:
+        print(f"Error parsing data: {e}")
+        return None, None, None, None, None

filter/tag.py

@@ -6,6 +6,7 @@ import sys
 import tty
 import termios
 import argparse
+from db_utils import fetch_entry_data, parse_entry_data
 
 # 数据库路径
 DATABASE_PATH = "./data/main.db"
@@ -45,42 +46,12 @@ def fetch_random_aids(db_path, num_entries=10, start_from=None):
     conn.close()
     return aids
 
-def fetch_entry_data(db_path, aid):
-    """
-    根据aid从数据库中加载data
-    """
-    conn = sqlite3.connect(db_path)
-    cursor = conn.cursor()
-    cursor.execute("SELECT data FROM bili_info_crawl WHERE aid = ?", (aid,))
-    fet = cursor.fetchone()
-    if fet:
-        data = fet[0]
-    else:
-        data = None
-    conn.close()
-    return data
-
-def parse_entry_data(data):
-    """
-    解析JSON数据，提取视频标题、简介、标签、作者简介
-    """
-    try:
-        obj = json.loads(data)
-        title = obj["View"]["title"]
-        description = obj["View"]["desc"]
-        tags = [tag["tag_name"] for tag in obj["Tags"] if tag["tag_type"] in ["old_channel", "topic"]]
-        author_info = obj["Card"]["card"]["name"] + ": " + obj["Card"]["card"]["sign"]
-        url = "https://www.bilibili.com/video/" + obj["View"]["bvid"]
-        return title, description, tags, author_info, url
-    except (TypeError, json.JSONDecodeError) as e:
-        print(f"Error parsing data: {e}")
-        return None, None, None, None, None
-
 def label_entries(db_path, aids):
     """
     标注工具：展示条目信息，等待用户输入标签
     """
     labeled_data = []
+    label_counts = {0: 0, 1: 0, 2: 0}
     for aid in aids:
         data = fetch_entry_data(db_path, aid)
         title, description, tags, author_info, url = parse_entry_data(data)
@@ -88,6 +59,7 @@ def label_entries(db_path, aids):
             continue
         # 展示信息
         os.system("clear")
+        print(f"Count: {label_counts[0]}, {label_counts[1]}, {label_counts[2]}")
         print(f"AID: {aid}")
         print(f"URL: {url}")
         print(f"Title: {title}")
@@ -105,6 +77,8 @@ def label_entries(db_path, aids):
             continue
         if label == "q":  # 退出
             break
+        # 更新计数
+        label_counts[int(label)] += 1
         # 保存标注结果
         labeled_data.append({
             "aid": aid,

filter/test.py

@@ -1,70 +1,173 @@
-import os, json
+import os
+import json
 import random
-os.environ["PYTORCH_ENABLE_MPS_FALLBACK"]="1"
+os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
 import torch
 from modelV3_4 import VideoClassifierV3_4
 from sentence_transformers import SentenceTransformer
-from tag import getch
+import sys
+import tty
+import termios
+from db_utils import fetch_entry_data, parse_entry_data
 
-def predict(json_input):
+DATABASE_PATH = "./data/main.db"
-    # 加载模型
+BATCH_SIZE = 50  # 动态加载批次大小
-    model = VideoClassifierV3_4()
-    model.load_state_dict(torch.load('./filter/checkpoints/best_model_V3.8.pt'))
-    model.eval()
 
-    # 加载SentenceTransformer
+class LabelingSystem:
-    sentence_transformer = SentenceTransformer("Thaweewat/jina-embedding-v3-m2v-1024")
+    def __init__(self):
+        # 初始化模型
+        self.model = VideoClassifierV3_4()
+        self.model.load_state_dict(torch.load('./filter/checkpoints/best_model_V3.8.pt'))
+        self.model.eval()
+        self.sentence_transformer = SentenceTransformer("Thaweewat/jina-embedding-v3-m2v-1024")
         
-    input_texts = {
+        # 数据相关
-        "title": [json_input["title"]],
+        self.existing_entries = self._load_existing_entries()
-        "description": [json_input["description"]],
+        self.existing_aids = set(entry['aid'] for entry in self.existing_entries)
-        "tags": [" ".join(json_input["tags"])],
+        self.candidate_pool = []
-        "author_info": [json_input["author_info"]]
+        self.history = []
+        self.current_index = -1  # -1表示未开始
+        
+        # 初始化第一批数据
+        self._load_more_candidates()
+
+    def _save_entry(self, entry):
+        """保存或更新条目"""
+        # 查找是否已存在
+        existing_index = next((i for i, e in enumerate(self.existing_entries) 
+                             if e['aid'] == entry['aid']), None)
+        
+        # 更新或添加条目
+        if existing_index is not None:
+            self.existing_entries[existing_index] = entry
+        else:
+            self.existing_entries.append(entry)
+        
+        # 重写整个文件
+        with open("./data/filter/real_test.jsonl", "w") as fp:
+            for entry in self.existing_entries:
+                fp.write(json.dumps(entry, ensure_ascii=False) + "\n")
+
+    def _load_existing_entries(self):
+        """加载已有条目"""
+        if not os.path.exists("./data/filter/real_test.jsonl"):
+            return []
+        
+        with open("./data/filter/real_test.jsonl", "r") as fp:
+            return [json.loads(line) for line in fp]
+
+    def _load_more_candidates(self):
+        """动态加载更多候选数据"""
+        with open('data/filter/model_predicted.jsonl', 'r') as fp:
+            new_candidates = []
+            for line in fp:
+                entry = json.loads(line)
+                if entry['aid'] not in self.existing_aids:
+                    new_candidates.append(entry['aid'])
+        
+        # 随机打乱后取批次
+        random.shuffle(new_candidates)
+        self.candidate_pool.extend(new_candidates[:BATCH_SIZE])
+        del new_candidates[:BATCH_SIZE]  # 释放内存
+
+    def _get_entry_details(self, aid):
+        """获取条目详细信息并预测模型标签"""
+        # 获取元数据
+        title, description, tags, author_info, url = parse_entry_data(
+            fetch_entry_data(DATABASE_PATH, aid)
+        )
+        
+        # 模型预测
+        with torch.no_grad():
+            logits = self.model(
+                input_texts={
+                    "title": [title],
+                    "description": [description],
+                    "tags": [" ".join(tags)],
+                    "author_info": [author_info]
+                },
+                sentence_transformer=self.sentence_transformer
+            )
+            model_label = torch.argmax(logits, dim=1).item()
+        
+        return {
+            'aid': aid,
+            'title': title,
+            'description': description,
+            'tags': tags,
+            'author_info': author_info,
+            'url': url,
+            'model_label': model_label,
+            'user_label': None
         }
 
-    # 预测
+    def _display_entry(self, entry):
-    with torch.no_grad():
+        """显示条目信息"""
-        logits = model(
+        os.system("clear")
-            input_texts=input_texts,
+        print(f"AID: {entry['aid']}")
-            sentence_transformer=sentence_transformer
+        print(f"URL: {entry['url']}")
-        )
+        print(f"Title: {entry['title']}")
-        pred = torch.argmax(logits, dim=1).item()
+        print(f"Tags: {', '.join(entry['tags'])}")
+        print(f"Author Info: {entry['author_info']}")
+        print(f"Description: {entry['description']}")
+        print(f"\nModel Prediction: {entry['model_label']}")
+        if entry['user_label'] is not None:
+            print(f"Your Label: {entry['user_label']}")
 
-    return pred
+    def run(self):
+        while True:
+            # 处理当前条目
+            if self.current_index < 0:
+                self.current_index = 0
+                
+            if self.current_index >= len(self.history):
+                if not self.candidate_pool:
+                    self._load_more_candidates()
+                    if not self.candidate_pool:
+                        print("\nAll entries processed!")
+                        return
+                
+                # 处理新条目
+                aid = self.candidate_pool.pop(0)
+                entry = self._get_entry_details(aid)
+                self.history.append(entry)
+                self.current_index = len(self.history) - 1
+
+            current_entry = self.history[self.current_index]
+            self._display_entry(current_entry)
+
+            # 获取用户输入
+            print("\nLabel (0/1/2, s=skip, ←↑/→↓=nav, q=quit): ", end="", flush=True)
+            cmd = getch().lower()
+
+            # 处理导航命令
+            if cmd in ['left', 'up']:
+                self.current_index = max(0, self.current_index - 1)
+            elif cmd in ['right', 'down']:
+                self.current_index += 1
+            elif cmd in ('0', '1', '2'):
+                current_entry['human'] = int(cmd)
+                self._save_entry(current_entry)
+                self.current_index += 1  # 自动前进
+            elif cmd == 's':
+                self.current_index += 1  # 跳过
+            elif cmd == 'q':
+                return
+
+def getch():
+    """支持方向键检测的输入函数"""
+    fd = sys.stdin.fileno()
+    old_settings = termios.tcgetattr(fd)
+    try:
+        tty.setraw(fd)
+        ch = sys.stdin.read(1)
+        if ch == '\x1b':
+            seq = sys.stdin.read(2)
+            return {'[A': 'up', '[B': 'down', '[C': 'right', '[D': 'left'}.get(seq, 'unknown')
+        return ch
+    finally:
+        termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)
 
 if __name__ == "__main__":
-    with open('data/filter/model_predicted.jsonl', 'r') as fp:
+    labeling_system = LabelingSystem()
-        data = [json.loads(line) for line in fp.readlines()]
+    labeling_system.run()
-    sampled = random.sample(data, min(200, len(data)))
-    test_data = []
-    for sample in sampled:
-        label = sample['label']
-        os.system("clear")
-        print(f"AID: {sample['aid']}")
-        print(f"Title: {sample['title']}")
-        print(f"Tags: {', '.join(sample['tags'])}")
-        print(f"Author Info: {sample['author_info']}")
-        print(f"Description: {sample['description']}")
-        # 等待用户输入
-        while True:
-            print("Label (0 or 1 or 2, s to skip, q to quit): ", end="", flush=True)
-            real_label = getch().lower()
-            if real_label in ["0", "1", "2", "s", "q"]:
-                break
-            print("\nInvalid input. Please enter 0, 1, 2, s or q.")
-        if real_label == "s":  # 跳过
-            continue
-        if real_label == "q":  # 退出
-            break
-        test_data.append({
-            "aid": sample['aid'],
-            "title": sample['title'],
-            "tags": sample['tags'],
-            "author_info": sample['author_info'],
-            "description": sample['description'],
-            "model": label,
-            "human": int(real_label)
-        })
-    
-    with open("./data/filter/real_test.jsonl", "a") as fp:
-        fp.writelines([json.dumps(item, ensure_ascii=False) + "\n" for item in test_data])