alikia2x/sparkast
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

ref: remove NLP

Browse Source 
improve: about page
This commit is contained in:
alikia2x (寒寒) 2025-01-20 04:31:10 +08:00
parent f7de462a28
commit 42cc96a0e3
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
29 changed files with 380 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
backend/nlp/.ipynb_checkpoints/Text Classification-checkpoint.ipynb
View File

@ -1,33 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

87
backend/nlp/Text Classification.ipynb
View File

@ -1,87 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 14,
"id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0mRequirement already satisfied: huggingface_hub in /opt/homebrew/lib/python3.11/site-packages (0.20.3)\n",
"Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n",
"Requirement already satisfied: requests in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n",
"Requirement already satisfied: tqdm>=4.42.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.8.0)\n",
"Requirement already satisfied: packaging>=20.9 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.3.2)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2023.11.17)\n",
"\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip install huggingface_hub"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "e82bbe25-4b14-4aca-8761-bad2a63fccd2",
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'huggingface_hub'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhuggingface_hub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m notebook_login\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'huggingface_hub'"
]
}
],
"source": [
"from huggingface_hub import notebook_login"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

BIN
bun.lockb
View File

Binary file not shown.

87
components/onesearch/SuggestionItem.tsx Normal file
View File

@ -0,0 +1,87 @@
import { SuggestionItem } from "global";
import PlainSearch from "./plainSearch";
import LinkSuggestion from "./link";
import PlainText from "./plainText";
export default function SuggestionComponent({
s,
i,
selected,
devMode,
engineName,
t
}: {
s: SuggestionItem;
i: number;
selected: number;
devMode: boolean;
engineName: string;
t: any;
}) {
if (s.suggestion.trim() === "") return null;
if (s.type === "default") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}&nbsp;
<span className="text-zinc-700 dark:text-zinc-400 text-sm">
{t("search.search-help-text", { engine: engineName })}
</span>
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "QUERY") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "NAVIGATION" || s.type === "default-link" || s.type === "link") {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
} else if (s.type === "text") {
return (
<PlainText key={i} selected={i == selected}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
<p>{s.suggestion}</p>
{devMode && (
<span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainText>
);
} else if (s.type === "inpage-link") {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected} inPage={true}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
}
return null;
}

4
components/onesearch/handleEnter.ts
View File

@ -1,11 +1,11 @@
import { settingsType, suggestionItem } from "global"; import { settingsType, SuggestionItem } from "global";
import copyToClipboard from "lib/copy"; import copyToClipboard from "lib/copy";
import { normalizeURL } from "lib/normalizeURL"; import { normalizeURL } from "lib/normalizeURL";
import search from "lib/search"; import search from "lib/search";
export default function ( export default function (
index: number, index: number,
suggestion: suggestionItem[], suggestion: SuggestionItem[],
_query: string, _query: string,
settings: settingsType, settings: settingsType,
searchBoxRef: React.RefObject<HTMLInputElement> searchBoxRef: React.RefObject<HTMLInputElement>

4
components/onesearch/handleNLUResult.ts
View File

@ -1,10 +1,10 @@
import { suggestionItem } from "global"; import { SuggestionItem } from "global";
import { findClosestDateIndex } from "lib/weather/getCurrentWeather"; import { findClosestDateIndex } from "lib/weather/getCurrentWeather";
import { getLocationNative } from "lib/weather/getLocation"; import { getLocationNative } from "lib/weather/getLocation";
import { getWeather } from "lib/weather/getWeather"; import { getWeather } from "lib/weather/getWeather";
import { WMOCodeTable } from "lib/weather/wmocode"; import { WMOCodeTable } from "lib/weather/wmocode";
type UpdateSuggestionFunction = (data: suggestionItem[]) => void; type UpdateSuggestionFunction = (data: SuggestionItem[]) => void;
export function handleNLUResult(result: any, updateSuggestion: UpdateSuggestionFunction) { export function handleNLUResult(result: any, updateSuggestion: UpdateSuggestionFunction) {
if (result.intent == "weather.summary") { if (result.intent == "weather.summary") {

187
components/onesearch/onesearch.tsx
View File

@ -1,39 +1,22 @@
import { useEffect, useRef, useState } from "react"; import { useEffect, useRef } from "react";
import SuggestionBox from "./suggestionBox"; import SuggestionBox from "./suggestionBox";
import { queryAtom } from "lib/state/query"; import { queryAtom } from "lib/state/query";
import { suggestionItem, suggestionsResponse } from "global"; import { SuggestionItem, SuggestionsResponse } from "global";
import getSearchEngineName from "lib/onesearch/getSearchEngineName"; import getSearchEngineName from "lib/onesearch/getSearchEngineName";
import PlainSearch from "./plainSearch";
import { suggestionAtom } from "lib/state/suggestion"; import { suggestionAtom } from "lib/state/suggestion";
import validLink from "lib/url/validLink"; import validLink from "lib/url/validLink";
import LinkSuggestion from "./link";
import { selectedSuggestionAtom } from "lib/state/suggestionSelection"; import { selectedSuggestionAtom } from "lib/state/suggestionSelection";
import { settingsAtom } from "lib/state/settings"; import { settingsAtom } from "lib/state/settings";
import PlainText from "./plainText";
import { sendError } from "lib/feedback/sendError"; import { sendError } from "lib/feedback/sendError";
import { handleNLUResult } from "./handleNLUResult";
import * as ort from "onnxruntime-web";
import { useAtom, useAtomValue } from "jotai"; import { useAtom, useAtomValue } from "jotai";
import i18next from "i18next"; import i18next from "i18next";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { keywordSuggestion } from "lib/onesearch/keywordSuggestion"; import { keywordSuggestion } from "lib/onesearch/keywordSuggestion";
import tokenize from "lib/nlp/tokenize/tokenizer"; import { searchboxLastInputAtom } from "lib/state/searchboxLastInput"
import { getEmbedding, getEmbeddingLayer } from "lib/nlp/getEmbedding"; import SuggestionComponent from "./SuggestionItem.tsx";
import { loadVocab } from "lib/nlp/tokenize/loadVocab";
import BPETokenizer from "lib/nlp/tokenize/BPEtokenizer";
import energyScore from "lib/nlp/energyScore";
import bytesToUnicode from "lib/nlp/tokenize/bytesToUnicode";
import { searchboxLastInputAtom } from "lib/state/searchboxLastInput";
interface EmbeddingLayer {
[key: number]: Float32Array<ArrayBufferLike>;
}
export default function OneSearch() { export default function OneSearch() {
const [suggestion, setFinalSuggetsion] = useAtom(suggestionAtom); const [suggestion, setFinalSuggestion] = useAtom(suggestionAtom);
const [embeddingLayer, setEmbeddingLayer] = useState<EmbeddingLayer | null>(null);
const [NLUsession, setNLUsession] = useState<ort.InferenceSession | null>(null);
const [tokenizer, setTokenizer] = useState<BPETokenizer | null>(null);
const lastInput = useAtomValue(searchboxLastInputAtom); const lastInput = useAtomValue(searchboxLastInputAtom);
const lastRequestTimeRef = useRef(0); const lastRequestTimeRef = useRef(0);
const selected = useAtomValue(selectedSuggestionAtom); const selected = useAtomValue(selectedSuggestionAtom);
@ -53,9 +36,9 @@ export default function OneSearch() {
} }
fetch(`/api/v1/suggestion?q=${query}&l=${lang}&t=${time}&engine=${engine}`) fetch(`/api/v1/suggestion?q=${query}&l=${lang}&t=${time}&engine=${engine}`)
.then((res) => res.json()) .then((res) => res.json())
.then((data: suggestionsResponse) => { .then((data: SuggestionsResponse) => {
try { try {
const suggestionToUpdate: suggestionItem[] = data.suggestions; const suggestionToUpdate: SuggestionItem[] = data.suggestions;
if (data.time > lastRequestTimeRef.current) { if (data.time > lastRequestTimeRef.current) {
cleanSuggestion("NAVIGATION", "QUERY"); cleanSuggestion("NAVIGATION", "QUERY");
lastRequestTimeRef.current = data.time; lastRequestTimeRef.current = data.time;
@ -73,8 +56,8 @@ export default function OneSearch() {
}); });
}, [lastInput]); }, [lastInput]);
function updateSuggestion(data: suggestionItem[]) { function updateSuggestion(data: SuggestionItem[]) {
setFinalSuggetsion((cur: suggestionItem[]) => { setFinalSuggestion((cur: SuggestionItem[]) => {
const types: string[] = []; const types: string[] = [];
for (const sug of data) { for (const sug of data) {
if (!types.includes(sug.type)) types.push(sug.type); if (!types.includes(sug.type)) types.push(sug.type);
@ -91,7 +74,7 @@ export default function OneSearch() {
} }
function cleanSuggestion(...types: string[]) { function cleanSuggestion(...types: string[]) {
setFinalSuggetsion((suggestion: suggestionItem[]) => { setFinalSuggestion((suggestion: SuggestionItem[]) => {
return suggestion.filter((item) => { return suggestion.filter((item) => {
return !types.includes(item.type); return !types.includes(item.type);
}); });
@ -99,52 +82,7 @@ export default function OneSearch() {
} }
useEffect(() => { useEffect(() => {
if (embeddingLayer !== null) return; cleanSuggestion("default-link", "default", "text", "link", "inpage-link");
const embedding_file = "/model/token_embeddings.bin";
(async function () {
const result = await fetch(embedding_file);
const arrBuf = await result.arrayBuffer();
const embeddingDict = getEmbeddingLayer(arrBuf);
setEmbeddingLayer(embeddingDict);
await loadModel("/model/NLU.onnx");
// if (!modelLoaded) {
// console.error("NLU model was not correctly loaded.")
// }
})();
}, []);
useEffect(() => {
if (tokenizer !== null) return;
(async function () {
await loadTokenizer();
})();
}, []);
async function loadModel(modelPath: string) {
ort.env.wasm.wasmPaths = "/onnx/";
const session = await ort.InferenceSession.create(modelPath);
setNLUsession(session);
}
async function loadTokenizer() {
const vocab = await loadVocab();
const tokenizer = new BPETokenizer(vocab);
setTokenizer(tokenizer);
}
async function getNLUResult(query: string) {
if (embeddingLayer === null || NLUsession === null || tokenizer == null) return;
const tokenIds = await tokenize(bytesToUnicode(query), tokenizer);
const embeddings = getEmbedding(tokenIds, embeddingLayer, 64);
const inputTensor = new ort.Tensor("float32", embeddings, [1, 64, 96]);
const feeds = { input: inputTensor };
const results = await NLUsession.run(feeds);
return results;
}
useEffect(() => {
cleanSuggestion("default-link", "default", "text", "link");
if (validLink(query)) { if (validLink(query)) {
updateSuggestion([ updateSuggestion([
{ {
@ -164,105 +102,24 @@ export default function OneSearch() {
} }
]); ]);
} }
if (keywordSuggestion(query) !== null) { if (keywordSuggestion(query) !== null) {
updateSuggestion([keywordSuggestion(query)!]); updateSuggestion([keywordSuggestion(query)!]);
} }
(async function () {
const result = await getNLUResult(query);
if (result === undefined) return;
const rawData = result.output.data;
const data: number[] = [];
for (let i=0;i<rawData.length;i++){
data.push(rawData[i] as number);
}
console.log(data, energyScore(data));
})();
}, [lastInput, engineName]); }, [lastInput, engineName]);
return ( return (
<SuggestionBox> <SuggestionBox>
{suggestion.map((s, i) => { {suggestion.map((s, i) => (
if (s.suggestion.trim() === "") return; <SuggestionComponent
if (s.type === "default") { key={i}
return ( s={s}
<PlainSearch key={i} query={s.suggestion} selected={i == selected}> i={i}
{s.suggestion}&nbsp; selected={selected}
<span className="text-zinc-700 dark:text-zinc-400 text-sm"> devMode={devMode}
{t("search.search-help-text", { engine: engineName })} engineName={engineName}
</span> t={t}
{devMode && ( />
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> ))}
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "QUERY") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (
s.type === "NAVIGATION" ||
s.type === "default-link" ||
s.type === "link"
) {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected}>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
} else if (s.type === "text") {
return (
<PlainText key={i} selected={i == selected}>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
<p>{s.suggestion}</p>
{devMode && (
<span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainText>
);
} else if (s.type === "inpage-link") {
return (
<LinkSuggestion
key={i}
query={s.suggestion}
selected={i == selected}
inPage={true}
>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
}
})}
</SuggestionBox> </SuggestionBox>
); );
} }

6
global.d.ts vendored
View File

@ -1,4 +1,4 @@
import { Suggestion } from "search-engine-autocomplete"; import React from "react";
interface settingsType extends object { interface settingsType extends object {
version: number; version: number;
@ -12,14 +12,14 @@ interface settingsType extends object {
}; };
} }
interface suggestionsResponse extends object { interface SuggestionsResponse extends object {
suggestions: Suggestion[]; suggestions: Suggestion[];
query: string; query: string;
verbatimRelevance: number; verbatimRelevance: number;
time: number; time: number;
} }
type suggestionItem = { interface SuggestionItem {
suggestion: string; suggestion: string;
type: string; type: string;
relativeRelevance?: number; relativeRelevance?: number;

21
i18n/en.json
View File

@ -33,7 +33,26 @@
"about": { "about": {
"title": "About sparkast", "title": "About sparkast",
"license": { "license": {
"view": "→ view" "title": "License",
"view": "→ view",
"text": "GPL 3.0"
},
"presented-by": "Presented By",
"backend-api-version": "Backend API Version",
"client-version": "Client API Version",
"oss": {
"title": "OSS Statement",
"view": "→ View"
} }
},
"license": {
"title": "License - sparkast",
"desc": "sparkast is open source software. It is licensed under the GPL 3.0 license. You can view the full license text below.",
"page-title": "License"
},
"oss_license": {
"title": "OSS Licenses - sparkast",
"desc": "This birth of this project would not have been possible without the following open source software. This page shows the licenses of the open source software used in sparkast.",
"page-title": "OSS Licenses"
} }
} }

93
i18n/zh.json
View File

@ -1,39 +1,58 @@
{ {
"404": { "404": {
"title": "页面未找到" "title": "页面未找到"
}, },
"search": { "search": {
"placeholder": "搜索或输入网址", "placeholder": "搜索或输入网址",
"engine-aria": "切换搜索引擎", "engine-aria": "切换搜索引擎",
"engine": { "engine": {
"google": "谷歌", "google": "谷歌",
"baidu": "百度", "baidu": "百度",
"bing": "必应", "bing": "必应",
"duckduckgo": "DuckDuckGo", "duckduckgo": "DuckDuckGo",
"yandex": "Yandex", "yandex": "Yandex",
"yahoo": "雅虎", "yahoo": "雅虎",
"ecosia": "Ecosia" "ecosia": "Ecosia"
}, },
"search-help-text": "用 {engine} 搜索" "search-help-text": "用 {engine} 搜索"
}, },
"tools": { "tools": {
"base64": { "base64": {
"title": "Base64 工具", "title": "Base64 工具",
"decode": "解码", "decode": "解码",
"encode": "编码", "encode": "编码",
"result": "结果： ", "result": "结果： ",
"copy": "复制", "copy": "复制",
"copied": "已复制" "copied": "已复制"
} }
}, },
"notfound": { "notfound": {
"desc": "请检查网址是否出错。 <br/>如果你从星火主页跳转到这里，<br/> 请 <a style=\"text-decoration:underline;\" href=\"mailto:contact@alikia2x.com\">联系我们</a>", "desc": "请检查网址是否出错。 <br/>如果你从星火主页跳转到这里，<br/> 请 <a style=\"text-decoration:underline;\" href=\"mailto:contact@alikia2x.com\">联系我们</a>",
"title": "网页不存在" "title": "网页不存在"
}, },
"about": { "about": {
"title": "关于 sparkast", "title": "关于 sparkast",
"license": { "oss": {
"view": "→ 查看" "title": "开源软件声明",
} "view": "→ 查看"
} },
"license": {
"title": "许可证",
"text": "GPL 3.0",
"view": "→ 查看"
},
"presented-by": "出品",
"backend-api-version": "后端 API 版本",
"client-version": "客户端 API 版本"
},
"license": {
"title": "开源许可证 - sparkast",
"page-title": "开源许可证",
"desc": "sparkast 是开源软件。它根据 GPL 3.0 许可证获得许可。您可以在下面查看完整的许可文本。"
},
"oss_license": {
"title": "开源软件声明 - sparkast",
"page-title": "开源软件声明",
"desc": "本项目的诞生离不开开源软件。此页面显示 sparkast 中使用的开源软件的许可证。"
}
} }

6
index.html
View File

@ -1,10 +1,12 @@
<!doctype html> <!doctype html>
<html> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<link rel="icon" href="/favicon.ico" /> <link rel="icon" href="/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>SparkHome</title> <meta property="og:title" content="sparkast">
<meta property="og:description" content="Your ultimate productivity tool that gives you the answer to all within a single search box.">
<title>sparkast</title>
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

6
lib/onesearch/handleEnter.ts
View File

@ -1,11 +1,11 @@
import { settingsType, suggestionItem } from "global"; import { settingsType, SuggestionItem } from "global";
import copyToClipboard from "lib/copy"; import copyToClipboard from "lib/copy";
import { normalizeURL } from "lib/normalizeURL"; import { normalizeURL } from "lib/normalizeURL";
import search from "lib/search"; import search from "lib/search";
export default function ( export default function (
index: number, index: number,
suggestion: suggestionItem[], suggestion: SuggestionItem[],
_query: string, _query: string,
settings: settingsType, settings: settingsType,
searchBoxRef: React.RefObject<HTMLInputElement> searchBoxRef: React.RefObject<HTMLInputElement>
@ -23,6 +23,6 @@ export default function (
} else if (selected.type === "link") { } else if (selected.type === "link") {
window.open(normalizeURL(selected.suggestion)); window.open(normalizeURL(selected.suggestion));
} else if (selected.type === "inpage-link") { } else if (selected.type === "inpage-link") {
location.href = normalizeURL(selected.suggestion); location.href = normalizeURL(selected.suggestion, false);
} }
} }

6
lib/onesearch/keywordSuggestion.ts
View File

@ -1,4 +1,4 @@
import { suggestionItem } from "global"; import { SuggestionItem } from "global";
interface keywordLinkDict { interface keywordLinkDict {
[key: string]: string; [key: string]: string;
@ -15,7 +15,7 @@ const dict_cn: keywordLinkDict = {
export function keywordSuggestion(query: string) { export function keywordSuggestion(query: string) {
for (const keyword in dict_cn) { for (const keyword in dict_cn) {
if (query.includes(keyword)) { if (query.includes(keyword)) {
const result: suggestionItem = { const result: SuggestionItem = {
type: "inpage-link", type: "inpage-link",
suggestion: dict_cn[keyword], suggestion: dict_cn[keyword],
prompt: keyword, prompt: keyword,
@ -26,7 +26,7 @@ export function keywordSuggestion(query: string) {
} }
for (const keyword in dict_en) { for (const keyword in dict_en) {
if (query.includes(keyword)) { if (query.includes(keyword)) {
const result: suggestionItem = { const result: SuggestionItem = {
type: "inpage-link", type: "inpage-link",
suggestion: dict_en[keyword], suggestion: dict_en[keyword],
prompt: keyword, prompt: keyword,

4
lib/state/suggestion.ts
View File

@ -1,6 +1,6 @@
import { suggestionItem } from "global"; import { SuggestionItem } from "global";
import { atom } from "jotai"; import { atom } from "jotai";
const suggestionAtom = atom([] as suggestionItem[]); const suggestionAtom = atom([] as SuggestionItem[]);
export { suggestionAtom }; export { suggestionAtom };

2
lib/url/tldList.ts
View File

@ -1,4 +1,4 @@
import TLDtxt from "./tlds.txt?raw"; import TLDtxt from "./tlds.txt";
export function getTLD() { export function getTLD() {
return TLDtxt.split("\r\n").filter((line) => line[0] !== "#"); return TLDtxt.split("\r\n").filter((line) => line[0] !== "#");

5
lib/url/tlds.txt
View File

@ -1,4 +1,4 @@
# Version 2024071300, Last Updated Sat Jul 13 07:07:01 2024 UTC # Version 2025011900, Last Updated Sun Jan 19 07:07:01 2025 UTC
AAA AAA
AARP AARP
ABB ABB
@ -297,7 +297,6 @@ CY
CYMRU CYMRU
CYOU CYOU
CZ CZ
DABUR
DAD DAD
DANCE DANCE
DATA DATA
@ -1444,4 +1443,4 @@ ZIP
ZM ZM
ZONE ZONE
ZUERICH ZUERICH
ZW ZW

30
lib/url/validLink.ts
View File

@ -1,41 +1,33 @@
import { toASCII } from "tr46"; import { toASCII } from "tr46";
import { getTLD } from "./tldList"; import { getTLD } from "./tldList";
console.log(getTLD());
export default function validLink(link: string) { export default function validLink(link: string) {
let finalURL; let finalURL;
try { try {
const url = new URL(link); new URL(link);
finalURL = url;
return true; return true;
} catch (error) { } catch (error) {
// if the URL is invalid, try to add the protocol // if the URL is invalid, try to add the protocol
try { try {
const urlWithHTTP = new URL("http://" + link); finalURL = new URL("http://" + link);
finalURL = urlWithHTTP;
} catch (error) { } catch (error) {
return false; return false;
} }
} }
if (finalURL.host.endsWith(".")) return false; if (finalURL.host.endsWith(".")) return false;
if ( return validTLD(finalURL.host) ||
validTLD(finalURL.host) ||
isValidIPv6(link.slice(1, finalURL.host.length - 1)) || isValidIPv6(link.slice(1, finalURL.host.length - 1)) ||
isValidIPv4(link) isValidIPv4(link);
) {
return true;
}
return false;
} }
export function validTLD(domain: string): boolean { export function validTLD(domain: string): boolean {
if (!domain.includes(".")) return false; if (!domain.includes(".")) return false;
const tld = toASCII(domain.split(".").reverse()[0]); const tld = toASCII(domain.split(".").reverse()[0]);
const tldList = getTLD(); const tldList = getTLD();
if (tldList.includes(tld.toUpperCase())) { return !!tldList.includes(tld.toUpperCase());
return true;
} else {
return false;
}
} }
export function isValidIPv6(ip: string): boolean { export function isValidIPv6(ip: string): boolean {
@ -69,10 +61,8 @@ export function isValidIPv6(ip: string): boolean {
return false; return false;
} }
} }
if (doubleColonCount === 0 && groups !== 8) { return !(doubleColonCount === 0 && groups !== 8);
return false;
}
return true;
} }
export function isValidIPv4(ip: string): boolean { export function isValidIPv4(ip: string): boolean {

5
lib/version.ts
View File

@ -1,8 +1,5 @@
import * as pjson from "package.json"; import * as pjson from "package.json";
export default function getVersion() { export const clientVersion = pjson.version;
return pjson.version;
}
export const clientNLUVersion = 4;
export const apiVersion = 1; export const apiVersion = 1;

128
package.json
View File

@ -1,65 +1,67 @@
{ {
"name": "sparkast", "name": "sparkast",
"private": false, "private": false,
"version": "5.8.1", "version": "5.8.1",
"type": "module", "type": "module",
"scripts": { "scripts": {
"dev": "bun server.ts", "dev": "bun server.ts",
"build": "bun license-gen && tsc -b && vite build", "build": "bun license-gen && tsc -b && vite build",
"lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0",
"preview": "NODE_ENV=production bun server.ts", "preview": "NODE_ENV=production bun server.ts",
"license-gen": "bunx generate-license-file --input package.json --output lib/license.txt --overwrite", "license-gen": "bunx generate-license-file --input package.json --output lib/license.txt --overwrite",
"format": "prettier --write ." "format": "prettier --write ."
}, },
"dependencies": { "dependencies": {
"@alikia/search-complete": "^0.4.4", "@alikia/search-complete": "^0.4.4",
"@iconify/react": "^5.0.1", "@iconify/react": "^5.0.1",
"@nextui-org/react": "^2.4.2", "@nextui-org/react": "^2.4.2",
"@types/bun": "^1.1.6", "@types/bun": "^1.1.6",
"@types/express": "^4.17.21", "@types/express": "^4.17.21",
"@types/tr46": "^5.0.0", "@types/tr46": "^5.0.0",
"@xenova/transformers": "^2.17.2", "@xenova/transformers": "^2.17.2",
"cac": "^6.7.14", "cac": "^6.7.14",
"chalk": "^5.3.0", "chalk": "^5.3.0",
"express": "^4.19.2", "express": "^4.19.2",
"fflate": "^0.8.2", "fflate": "^0.8.2",
"framer-motion": "^11.2.12", "framer-motion": "^11.2.12",
"generate-license-file": "^3.5.1", "generate-license-file": "^3.5.1",
"i18next": "^23.11.5", "i18next": "^23.11.5",
"i18next-browser-languagedetector": "^8.0.0", "i18next-browser-languagedetector": "^8.0.0",
"i18next-icu": "^2.3.0", "i18next-icu": "^2.3.0",
"jest": "^29.7.0", "jest": "^29.7.0",
"jotai": "^2.8.3", "jotai": "^2.8.3",
"node-nlp": "^4.27.0", "node-nlp": "^4.27.0",
"onnxruntime-web": "^1.20.0-dev.20240925-a47254eaef", "onnxruntime-web": "^1.20.0-dev.20240925-a47254eaef",
"react": "^18.3.1", "react": "^18.3.1",
"react-dom": "^18.3.1", "react-dom": "^18.3.1",
"react-i18next": "^14.1.2", "react-helmet": "^6.1.0",
"react-router": "^6.23.1", "react-i18next": "^14.1.2",
"react-router-dom": "^6.23.1", "react-router": "^6.23.1",
"tr46": "^5.0.0", "react-router-dom": "^6.23.1",
"valid-url": "^1.0.9", "tr46": "^5.0.0",
"validate-color": "^2.2.4", "valid-url": "^1.0.9",
"vite-express": "^0.17.0" "validate-color": "^2.2.4",
}, "vite-express": "^0.17.0"
"devDependencies": { },
"@types/react": "^18.3.3", "devDependencies": {
"@types/react-dom": "^18.3.0", "@types/react": "^18.3.3",
"@types/valid-url": "^1.0.7", "@types/react-dom": "^18.3.0",
"@typescript-eslint/eslint-plugin": "^7.13.1", "@types/react-helmet": "^6.1.11",
"@typescript-eslint/parser": "^7.13.1", "@types/valid-url": "^1.0.7",
"@vitejs/plugin-react-swc": "^3.5.0", "@typescript-eslint/eslint-plugin": "^7.13.1",
"autoprefixer": "^10.4.19", "@typescript-eslint/parser": "^7.13.1",
"eslint": "^8.57.0", "@vitejs/plugin-react-swc": "^3.5.0",
"eslint-plugin-react-hooks": "^4.6.2", "autoprefixer": "^10.4.19",
"eslint-plugin-react-refresh": "^0.4.7", "eslint": "^8.57.0",
"postcss": "^8.4.38", "eslint-plugin-react-hooks": "^4.6.2",
"prettier": "^3.3.3", "eslint-plugin-react-refresh": "^0.4.7",
"tailwindcss": "^3.4.4", "postcss": "^8.4.38",
"typescript": "^5.2.2", "prettier": "^3.3.3",
"vite": "^5.3.1", "tailwindcss": "^3.4.4",
"vite-plugin-chunk-split": "^0.5.0", "typescript": "^5.2.2",
"vite-plugin-pages": "^0.32.2", "vite": "^5.3.1",
"vite-tsconfig-paths": "^4.3.2" "vite-plugin-chunk-split": "^0.5.0",
} "vite-plugin-pages": "^0.32.2",
"vite-tsconfig-paths": "^4.3.2"
}
} }

66
pages/about/index.tsx
View File

@ -1,8 +1,38 @@
import useDarkMode from "lib/darkModeHook"; import useDarkMode from "lib/darkModeHook";
import getVersion, { apiVersion, clientNLUVersion } from "lib/version"; import { apiVersion, clientVersion } from "lib/version";
import AboutLayout from "./layout"; import AboutLayout from "./layout";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
function License() {
const { t } = useTranslation();
return (
<>
<p className="flex items-center my-4">
<span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t("about.oss.title")}
</span>
<span
className="relative font-bold px-2 py-1 rounded-md text-nowrap underline
bg-green-600 text-white"
>
<a href="/about/oss-licenses">{t("about.oss.view")}</a>
</span>
</p>
<p className="flex items-center my-4">
<span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t("about.license.title")}
</span>
<span
className="relative font-bold px-2 py-1 rounded-md text-nowrap underline
bg-red-500 text-white"
>
<a href="/about/license">{t("about.license.text")}</a>
</span>
</p>
</>
);
}
export default function AboutPage() { export default function AboutPage() {
const darkMode = useDarkMode(); const darkMode = useDarkMode();
const { t } = useTranslation(); const { t } = useTranslation();
@ -26,47 +56,41 @@ export default function AboutPage() {
</div> </div>
</div> </div>
<Version title="Frontend Version" version={getVersion()} versionClass="bg-red-500" />
<Version <Version
title="Browser NLU Model Version" title={t("about.backend-api-version")}
version={"Build " + clientNLUVersion} version={"/api/v" + apiVersion}
versionClass="bg-purple-500" versionClass="bg-purple-500"
/> />
<Version <Version
title="Backend API Version" title={t("about.client-version")}
version={"/api/v" + apiVersion} version={clientVersion}
versionClass="bg-orange-500" versionClass="bg-orange-500"
/> />
<p className="flex items-center my-3">
<span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">License</span>
<span
className="relative px-2 py-1 text-sm font-bold rounded-md text-nowrap underline
bg-green-600 text-white"
>
<a href="/about/license">{t("about.license.view")}</a>
</span>
</p>
<p className="relative font-bold text-2xl mt-12">Presented By</p> <License />
<p className="relative font-bold text-2xl mt-12">{t("about.presented-by")}</p>
{!darkMode && ( {!darkMode && (
<img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" /> <img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" />
)} )}
{darkMode && ( {darkMode && (
<img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-56 mt-6" /> <img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-64 mt-6" />
)} )}
</AboutLayout> </AboutLayout>
); );
} }
function Version(props: { title: string; version: string; versionClass?: string }) { function Version(props: { title: string; version: string; versionClass?: string }) {
document.title = "About SparkHome";
const { t } = useTranslation(); const { t } = useTranslation();
return ( return (
<p className="flex items-center my-3"> <p className="flex items-center my-4">
<span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">{t(props.title)}</span> <span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t(props.title)}
</span>
<span <span
className={ className={
"relative px-2 py-1 text-sm font-bold rounded-md text-nowrap text-white " + "relative px-2 py-1 font-bold rounded-md text-nowrap text-white " +
props.versionClass || "" props.versionClass || ""
} }
> >

8
pages/about/layout.tsx
View File

@ -1,6 +1,14 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Helmet } from "react-helmet";
export default function AboutLayout({ children }: { children: React.ReactNode }) { export default function AboutLayout({ children }: { children: React.ReactNode }) {
const { t } = useTranslation();
return ( return (
<div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]"> <div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]">
<Helmet>
<title>{t('about.title')}</title>
</Helmet>
<main <main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4 className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12 px-3 md:px-0" pt-12 px-3 md:px-0"

11
pages/about/license/index.tsx → pages/about/license/OSS.tsx
View File

@ -1,13 +1,20 @@
import LICENSE from "lib/license.txt?raw"; import LICENSE from "lib/license.txt?raw";
import { Helmet } from "react-helmet";
import { useTranslation } from "react-i18next";
export default function LicensePage() { export default function OSSLicensesPage() {
const { t } = useTranslation();
return ( return (
<div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden"> <div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden">
<Helmet>
<title>{t("oss_license.title")}</title>
</Helmet>
<main <main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4 className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12" pt-12"
> >
<h1 className="text-4xl font-bold mb-6">LICENSE</h1> <h1 className="text-4xl font-bold mb-6">{t('oss_license.page-title')}</h1>
<p className="text-lg mb-8">{t('oss_license.desc')}</p>
<div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div> <div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div>
</main> </main>
</div> </div>

22
pages/about/license/ThisProject.tsx Normal file
View File

@ -0,0 +1,22 @@
import LICENSE from "LICENSE?raw";
import { useTranslation } from "react-i18next";
import { Helmet } from "react-helmet";
export default function ThisProjectLicensePage() {
const { t } = useTranslation();
return (
<div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden">
<Helmet>
<title>{t("license.title")}</title>
</Helmet>
<main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12"
>
<h1 className="text-4xl font-bold mb-6">{t('license.page-title')}</h1>
<p className="text-lg mb-8">{t('license.desc')}</p>
<div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div>
</main>
</div>
);
}

1
pages/index.tsx
View File

@ -13,6 +13,7 @@ export default function Homepage() {
return ( return (
<div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]"> <div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]">
<title>sparkast</title>
<Background /> <Background />
<EngineSelector <EngineSelector

19
src/app.tsx
View File

@ -3,7 +3,8 @@ import { createBrowserRouter, RouterProvider } from "react-router-dom";
import "./i18n"; import "./i18n";
import Homepage from "pages"; import Homepage from "pages";
import AboutPage from "pages/about"; import AboutPage from "pages/about";
import LicensePage from "pages/about/license"; import OSSLicensesPage from "pages/about/license/OSS";
import ThisProjectLicensePage from "../pages/about/license/ThisProject.tsx";
const router = createBrowserRouter([ const router = createBrowserRouter([
{ {
@ -11,14 +12,16 @@ const router = createBrowserRouter([
element: <Homepage /> element: <Homepage />
}, },
{ {
path: "about", path: "/about",
element: <AboutPage />, element: <AboutPage />,
children: [ },
{ {
path: "license", path: "/about/oss-licenses",
element: <LicensePage /> element: <OSSLicensesPage />
} },
] {
path: "/about/license",
element: <ThisProjectLicensePage />
} }
]); ]);

6
test/bytesToUnicode.test.ts
View File

@ -1,6 +0,0 @@
import { expect, test } from "bun:test";
import bytesToUnicodes from "../lib/nlp/tokenize/bytesToUnicode";
test("bytesToUnicode: test", () => {
expect(bytesToUnicodes("Hello 你好")).toEqual("HelloĠä½łå¥½");
});

0
test/getEmbeddings.test.ts
View File

6
test/unicodeToBytes.test.ts
View File

@ -1,6 +0,0 @@
import { expect, test } from "bun:test";
import unicodeToBytes from "../lib/nlp/tokenize/unicodeToBytes";
test("unicodeToBytes: test", () => {
expect(unicodeToBytes("HelloĠä½łå¥½")).toEqual("Hello 你好");
});

1
test/validLink.test.ts
View File

@ -60,6 +60,7 @@ describe("Check if the given TLD exist and assigned.", () => {
expect(validTLD("example.foo")).toBe(true); expect(validTLD("example.foo")).toBe(true);
expect(validTLD("example.bar")).toBe(true); expect(validTLD("example.bar")).toBe(true);
expect(validTLD("example.zip")).toBe(true); expect(validTLD("example.zip")).toBe(true);
expect(validTLD("xn--s8w913fdga.chn.moe")).toBe(true);
}); });
test("Exist but not assigned TLD", () => { test("Exist but not assigned TLD", () => {
expect(validTLD("example.active")).toBe(false); expect(validTLD("example.active")).toBe(false);