alikia2x/sparkast
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

ref: re-enable backend api, remove deprecated code

Browse Source
This commit is contained in:
alikia2x (寒寒) 2024-10-13 18:10:44 +08:00
parent a15694ada8
commit 1acb0a7f11
9 changed files with 16 additions and 403 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
backend/route.ts
View File

@ -1,21 +1,21 @@
import { Express } from "express";
//import { completeGoogle } from "search-engine-autocomplete";
import { completeGoogle } from "@alikia/search-complete";
export function configureBackendRoutes(app: Express) {
// app.get("/api/v1/suggestion", async (req, res) => {
// const query = req.query.q as string;
// const t = parseInt((req.query.t as string) || "0") || null;
// let language = (req.query.l as string) || "en-US";
app.get("/api/v1/suggestion", async (req, res) => {
const query = req.query.q as string;
const t = parseInt((req.query.t as string) || "0") || null;
let language = (req.query.l as string) || "en-US";
// try {
// const data = await completeGoogle(query, language);
// //logger.info({ type: "onesearch_search_autocomplete", query: query, data: data });
// res.json({ ...data, time: t });
// } catch (error) {
// //logger.error({ type: "onesearch_search_autocomplete_error", error: error.message });
// res.status(500).json({ error: "Internal Server Error" });
// }
// });
try {
const data = await completeGoogle(query, language);
// logger.info({ type: "onesearch_search_autocomplete", query: query, data: data });
res.json({ ...data, time: t });
} catch (error) {
// logger.error({ type: "onesearch_search_autocomplete_error", error: error.message });
res.status(500).json({ error: "Internal Server Error" });
}
});
app.get("/api/v1/ping", async (_, res) => {
res.status(200).json({ message: "pong" });
});

BIN
bun.lockb
View File

Binary file not shown.

2
components/onesearch/onesearch.tsx
View File

@ -10,7 +10,7 @@ import LinkSuggestion from "./link";
import { selectedSuggestionAtom } from "lib/state/suggestionSelection";
import { settingsAtom } from "lib/state/settings";
import PlainText from "./plainText";
import { sendError } from "lib/telemetering/sendError";
import { sendError } from "lib/feedback/sendError";
import { handleNLUResult } from "./handleNLUResult";
import * as ort from "onnxruntime-web";
import { useAtom, useAtomValue } from "jotai";

0
lib/telemetering/sendError.ts → lib/feedback/sendError.ts
View File

141
lib/nlp/data/en.json
View File

@ -1,141 +0,0 @@
{
"weather.summary": [
"how's the weather",
"What's going on with the weather?",
"Can you give me an update on the weather?",
"How's the forecast looking today?",
"Give me a summary of the current weather.",
"Can you tell me the current weather?",
"What is the weather situation at the moment?",
"Could you provide a quick weather update?",
"Is it raining or sunny outside?",
"What's the weather like right now?",
"Tell me the current weather conditions.",
"How about the weather today?",
"Is it a good day to be outside?",
"What should I expect in terms of weather today?",
"Is there any severe weather to be aware of?",
"Can you summarize today's weather forecast?",
"What's the weather looking like for the next few hours?",
"Is it going to stay this way all day?",
"Could you give me a brief overview of the weather?",
"What's the general weather situation in our area?",
"Is it cloudy or clear outside?",
"Any weather alerts I should know about?",
"How's the weather looking for outdoor activities?",
"What's the forecast saying for today's weather?",
"Is it going to be a warm day?",
"Are we expecting any storms today?",
"What's the weather condition outside my window?",
"Is it a typical day for this season in terms of weather?",
"how's the weather now?"
],
"weather.temp": [
"What's the temperature like right now?",
"Can you tell me the current temperature?",
"How hot is it outside?",
"What's the temperature supposed to be today?",
"What is the current temp outside?",
"Could you tell me the outdoor temperature?",
"Is it cold or warm outside?",
"What's the high temperature for today?",
"What's the low temperature expected tonight?",
"How does the temperature feel outside?",
"Is it going to get warmer or cooler today?",
"What's the temperature in the shade?",
"Can you provide the current temp in Celsius?",
"What's the temperature in Fahrenheit right now?",
"Is it too hot to be outside?",
"What's the temperature like in the morning?",
"How about the temperature in the evening?",
"Is it warm enough to go swimming?",
"What's the temperature in the city center?",
"Can you tell me the temp in the nearby area?",
"Is it below freezing outside?",
"What's the average temperature for today?",
"Is the temperature dropping or rising?",
"What should I wear considering the temperature?"
],
"base64.encode": [
"Please encode this data with base64: %s",
"I need to encode the following data in base64: %s",
"Could you encode this string using base64? %s",
"Convert this data to b64 encoding: %s",
"I want to encode this information with base64: %s",
"Help me encode this in base64: %s",
"Can you encode this data to base64 format? %s",
"b64 encode",
"base64 encode",
"encode base64 %s"
],
"base64.decode": [
"Please decode this base64 data: %s",
"I have a base64 encoded string that needs decoding: %s",
"Could you decode this base64 string for me? %s",
"Convert this base64 encoded data back to its original form: %s",
"I need to decode this base64 information: %s",
"Help me decode this base64 data: %s",
"Can you translate this base64 back to normal text? %s",
"b64 decode",
"base64 decode",
"decode base64 %s"
],
"url.encode": [
"Please encode this URL: %s",
"I need to encode this URL component: %s",
"Could you encode this part of the URL? %s",
"Convert this URL to its encoded form: %s",
"I want to encode this URL for safe transmission: %s",
"Help me encode this URL segment: %s",
"Can you encode this URL data? %s"
],
"url.decode": [
"Please decode this URL: %s",
"I have an encoded URL that needs decoding: %s",
"Could you decode this URL for me? %s",
"Convert this encoded URL back to its original form: %s",
"I need to decode this URL component: %s",
"Help me decode this URL segment: %s",
"Can you translate this encoded URL back to normal? %s"
],
"html.encode": [
"Please encode this HTML entity: %s",
"I need to encode this text to HTML entity: %s",
"Could you encode this as an HTML entity? %s",
"Convert this text to HTML entity encoding: %s",
"I want to encode this to prevent HTML interpretation: %s",
"Help me encode this into HTML entity: %s",
"Can you encode this for HTML usage? %s"
],
"html.decode": [
"Please decode this HTML entity: %s",
"I have an HTML entity that needs decoding: %s",
"Could you decode this HTML entity for me? %s",
"Convert this HTML entity back to its original text: %s",
"I need to decode this HTML entity to plain text: %s",
"Help me decode this HTML entity: %s",
"Can you translate this HTML entity back to normal text? %s"
],
"ai.command": [
"write a typescript helloworld code",
"Check the following content for grammar and clarity",
"Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option.",
"Improve this markdown content in asepcts like grammar and expression, for a GitHub repo README.",
"can u think of a short name of my package",
"simplify this code"
],
"ai.answer": [
"Which framework do you think is the most suitable for performance sensitive projects?"
],
"None": ["free weather api", "js get timezone", "how", "how's", "how's the"]
}

121
lib/nlp/data/zh.json
View File

@ -1,121 +0,0 @@
{
"weather.summary": [
"天气如何",
"现在的天气",
"今天的天气预报",
"现在的天气状况",
"今天天气怎么样",
"目前是什么天气",
"今天的天气概述",
"当前天气状况如何",
"今天会下雨吗",
"今天会下雪吗",
"今天晴天吗",
"今天的天气状况如何",
"现在外面是什么天气",
"今天天气好么",
"今天适合外出吗",
"今天的天气适宜做什么",
"今天有没有雾霾",
"今天的空气质量如何",
"今天的紫外线指数是多少",
"今天有没有大风",
"今天会不会很冷",
"今天的天气会变化吗",
"今天晚上的天气如何",
"今天夜里会下雨吗",
"今天的天气对出行有影响吗",
"今天的天气对运动有影响吗",
"今天的天气对工作有影响吗",
"今天的天气对旅游有影响吗",
"今天的天气对健康有影响吗"
],
"weather.temp": [
"现在的温度",
"现在多少度",
"外面有多热",
"明天热不热？",
"现在的气温是多少",
"今天最高温度是多少",
"今天最低温度是多少",
"现在外面感觉冷吗",
"现在需要穿外套吗",
"现在适合穿短袖吗",
"现在的温度适合外出吗",
"现在的温度适合运动吗",
"现在的温度适合睡觉吗",
"明天会比今天热吗",
"明天会比今天冷吗",
"今天的温度变化大吗",
"现在的温度适合开空调吗",
"现在的温度适合开暖气吗",
"室外的温度是多少",
"室内的温度是多少",
"现在的温度适合种植吗",
"现在的温度适合养宠物吗",
"现在的温度对健康有影响吗",
"现在的温度是否舒适",
"现在的温度是否适合工作"
],
"base64.encode": [
"请将数据使用base64编码：%s",
"需要将以下数据base64编码：%s",
"请将此字符串转为base64：%s",
"将数据转为base64编码：%s",
"信息base64编码：%s",
"请帮忙编码base64：%s",
"将数据编码为base64：%s"
],
"base64.decode": [
"请解码这个base64数据：%s",
"有base64编码字符串需要解码：%s",
"帮忙解码base64：%s",
"将base64编码转回原数据：%s",
"解码base64信息：%s",
"解码这个base64：%s",
"将base64转文本：%s"
],
"url.encode": [
"请编码这个URL：%s",
"URL部分需要编码：%s",
"请将URL部分编码：%s",
"URL编码转换：%s",
"安全传输需编码URL：%s",
"编码URL段：%s",
"URL数据编码：%s"
],
"url.decode": [
"请解码这个URL：%s",
"有URL编码需要解码：%s",
"解码这个URL：%s",
"URL编码转回原URL：%s",
"解码URL部分：%s",
"解码URL段：%s",
"URL编码转文本：%s"
],
"html.encode": [
"请编码HTML实体：%s",
"文本转为HTML实体：%s",
"编码为HTML实体：%s",
"文本HTML实体编码：%s",
"预防HTML解析编码：%s",
"HTML实体编码：%s",
"文本HTML使用编码：%s"
],
"html.decode": [
"请解码HTML实体：%s",
"HTML实体需要解码：%s",
"解码HTML实体：%s",
"HTML实体转回文本：%s",
"HTML实体解码：%s",
"解码HTML实体：%s",
"HTML实体转文本：%s"
],
"None": ["你好", "为什么计算机使用二进制", "什么是", "热", "怎么"]
}

55
lib/nlp/load.ts
View File

@ -1,55 +0,0 @@
// @ts-ignore
import { containerBootstrap } from "@nlpjs/core";
// @ts-ignore
import { Nlp } from "@nlpjs/nlp";
// @ts-ignore
import { NluManager, NluNeural } from "@nlpjs/nlu";
// @ts-ignore
import { LangEn } from "@nlpjs/lang-en-min";
// @ts-ignore
import { LangZh } from "@nlpjs/lang-zh";
import * as fflate from "fflate";
export interface NLUType {
manager: any;
inited: boolean;
loadIntentionModel(): Promise<void>;
init(): Promise<this>;
process(lang: string, text: string): Promise<any>;
}
export class NLU {
manager: any;
inited: boolean = false;
async loadIntentionModel() {
const container = await containerBootstrap();
container.use(Nlp);
container.use(LangEn);
container.use(LangZh);
container.use(NluNeural);
const manager = new NluManager({
container,
locales: ["en", "zh"],
nlu: {
useNoneFeature: true
}
});
const response = await fetch("/model");
const responseBuf = await response.arrayBuffer();
const compressed = new Uint8Array(responseBuf);
const decompressed = fflate.decompressSync(compressed);
const modelText = fflate.strFromU8(decompressed);
manager.fromJSON(JSON.parse(modelText));
this.manager = manager;
}
async init() {
await this.loadIntentionModel();
this.inited = true;
return this;
}
async process(lang: string, text: string): Promise<any> {
const actual = await this.manager.process(lang, text);
return actual;
}
}

71
lib/nlp/train.ts
View File

@ -1,71 +0,0 @@
// @ts-ignore
import { containerBootstrap } from "@nlpjs/core";
// @ts-ignore
import { Nlp } from "@nlpjs/nlp";
// @ts-ignore
import { NluManager, NluNeural } from "@nlpjs/nlu";
// @ts-ignore
import { LangEn } from "@nlpjs/lang-en-min";
// @ts-ignore
import { LangZh } from "@nlpjs/lang-zh";
import fs from "node:fs";
import * as fflate from "fflate";
let zh: TrainData = {};
let en: TrainData = {};
type TrainData = {
[key: string]: string[];
};
export async function trainIntentionModel() {
try {
const dataZH = fs.readFileSync("./lib/nlp/data/zh.json", "utf8");
const dataEN = fs.readFileSync("./lib/nlp/data/en.json", "utf8");
zh = JSON.parse(dataZH);
en = JSON.parse(dataEN);
} catch (err) {
console.error(err);
}
const container = await containerBootstrap();
container.use(Nlp);
container.use(LangEn);
container.use(LangZh);
container.use(NluNeural);
const manager = new NluManager({
container,
locales: ["en", "zh"],
nlu: {
useNoneFeature: true
}
});
// Adds the utterances and intents for the NLP
for (const key in zh) {
for (const value of zh[key]) {
manager.add("zh", value, key);
}
}
for (const key in en) {
for (const value of en[key]) {
manager.add("en", value, key);
}
}
await manager.train();
const resultModel = manager.toJSON();
const buf = fflate.strToU8(JSON.stringify(resultModel));
const gzipped = fflate.gzipSync(buf, {
filename: "model.json",
mtime: new Date().getTime()
});
fs.writeFileSync("./public/model", Buffer.from(gzipped));
}
trainIntentionModel();

1
package.json
View File

@ -12,6 +12,7 @@
"format": "prettier --write ."
},
"dependencies": {
"@alikia/search-complete": "^0.4.4",
"@iconify/react": "^5.0.1",
"@nextui-org/react": "^2.4.2",
"@types/bun": "^1.1.6",