alikia2x/sparkast
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases Wiki Activity

ref: remove NLP

Browse Source 
improve: about page
This commit is contained in:
alikia2x (寒寒) 2025-01-20 04:31:10 +08:00
parent f7de462a28
commit 42cc96a0e3
Signed by: alikia2x
GPG Key ID: 56209E0CCD8420C6
29 changed files with 380 additions and 474 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
backend/nlp/.ipynb_checkpoints/Text Classification-checkpoint.ipynb
View File

@ -1,33 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

87
backend/nlp/Text Classification.ipynb
View File

@ -1,87 +0,0 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 14,
"id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0mRequirement already satisfied: huggingface_hub in /opt/homebrew/lib/python3.11/site-packages (0.20.3)\n",
"Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n",
"Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n",
"Requirement already satisfied: requests in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n",
"Requirement already satisfied: tqdm>=4.42.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n",
"Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n",
"Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.8.0)\n",
"Requirement already satisfied: packaging>=20.9 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.3.2)\n",
"Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.0)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2023.11.17)\n",
"\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n",
"\u001b[0m"
]
}
],
"source": [
"!pip install huggingface_hub"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "e82bbe25-4b14-4aca-8761-bad2a63fccd2",
"metadata": {},
"outputs": [
{
"ename": "ModuleNotFoundError",
"evalue": "No module named 'huggingface_hub'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhuggingface_hub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m notebook_login\n",
"\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'huggingface_hub'"
]
}
],
"source": [
"from huggingface_hub import notebook_login"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

BIN
bun.lockb
View File

Binary file not shown.

87
components/onesearch/SuggestionItem.tsx Normal file
View File

@ -0,0 +1,87 @@
import { SuggestionItem } from "global";
import PlainSearch from "./plainSearch";
import LinkSuggestion from "./link";
import PlainText from "./plainText";
export default function SuggestionComponent({
s,
i,
selected,
devMode,
engineName,
t
}: {
s: SuggestionItem;
i: number;
selected: number;
devMode: boolean;
engineName: string;
t: any;
}) {
if (s.suggestion.trim() === "") return null;
if (s.type === "default") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}&nbsp;
<span className="text-zinc-700 dark:text-zinc-400 text-sm">
{t("search.search-help-text", { engine: engineName })}
</span>
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "QUERY") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "NAVIGATION" || s.type === "default-link" || s.type === "link") {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
} else if (s.type === "text") {
return (
<PlainText key={i} selected={i == selected}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
<p>{s.suggestion}</p>
{devMode && (
<span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainText>
);
} else if (s.type === "inpage-link") {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected} inPage={true}>
{s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
}
return null;
}

4
components/onesearch/handleEnter.ts
View File

@ -1,11 +1,11 @@
import { settingsType, suggestionItem } from "global";
import { settingsType, SuggestionItem } from "global";
import copyToClipboard from "lib/copy";
import { normalizeURL } from "lib/normalizeURL";
import search from "lib/search";
export default function (
index: number,
suggestion: suggestionItem[],
suggestion: SuggestionItem[],
_query: string,
settings: settingsType,
searchBoxRef: React.RefObject<HTMLInputElement>

4
components/onesearch/handleNLUResult.ts
View File

@ -1,10 +1,10 @@
import { suggestionItem } from "global";
import { SuggestionItem } from "global";
import { findClosestDateIndex } from "lib/weather/getCurrentWeather";
import { getLocationNative } from "lib/weather/getLocation";
import { getWeather } from "lib/weather/getWeather";
import { WMOCodeTable } from "lib/weather/wmocode";
type UpdateSuggestionFunction = (data: suggestionItem[]) => void;
type UpdateSuggestionFunction = (data: SuggestionItem[]) => void;
export function handleNLUResult(result: any, updateSuggestion: UpdateSuggestionFunction) {
if (result.intent == "weather.summary") {

185
components/onesearch/onesearch.tsx
View File

@ -1,39 +1,22 @@
import { useEffect, useRef, useState } from "react";
import { useEffect, useRef } from "react";
import SuggestionBox from "./suggestionBox";
import { queryAtom } from "lib/state/query";
import { suggestionItem, suggestionsResponse } from "global";
import { SuggestionItem, SuggestionsResponse } from "global";
import getSearchEngineName from "lib/onesearch/getSearchEngineName";
import PlainSearch from "./plainSearch";
import { suggestionAtom } from "lib/state/suggestion";
import validLink from "lib/url/validLink";
import LinkSuggestion from "./link";
import { selectedSuggestionAtom } from "lib/state/suggestionSelection";
import { settingsAtom } from "lib/state/settings";
import PlainText from "./plainText";
import { sendError } from "lib/feedback/sendError";
import { handleNLUResult } from "./handleNLUResult";
import * as ort from "onnxruntime-web";
import { useAtom, useAtomValue } from "jotai";
import i18next from "i18next";
import { useTranslation } from "react-i18next";
import { keywordSuggestion } from "lib/onesearch/keywordSuggestion";
import tokenize from "lib/nlp/tokenize/tokenizer";
import { getEmbedding, getEmbeddingLayer } from "lib/nlp/getEmbedding";
import { loadVocab } from "lib/nlp/tokenize/loadVocab";
import BPETokenizer from "lib/nlp/tokenize/BPEtokenizer";
import energyScore from "lib/nlp/energyScore";
import bytesToUnicode from "lib/nlp/tokenize/bytesToUnicode";
import { searchboxLastInputAtom } from "lib/state/searchboxLastInput";
interface EmbeddingLayer {
[key: number]: Float32Array<ArrayBufferLike>;
}
import { searchboxLastInputAtom } from "lib/state/searchboxLastInput"
import SuggestionComponent from "./SuggestionItem.tsx";
export default function OneSearch() {
const [suggestion, setFinalSuggetsion] = useAtom(suggestionAtom);
const [embeddingLayer, setEmbeddingLayer] = useState<EmbeddingLayer | null>(null);
const [NLUsession, setNLUsession] = useState<ort.InferenceSession | null>(null);
const [tokenizer, setTokenizer] = useState<BPETokenizer | null>(null);
const [suggestion, setFinalSuggestion] = useAtom(suggestionAtom);
const lastInput = useAtomValue(searchboxLastInputAtom);
const lastRequestTimeRef = useRef(0);
const selected = useAtomValue(selectedSuggestionAtom);
@ -53,9 +36,9 @@ export default function OneSearch() {
}
fetch(`/api/v1/suggestion?q=${query}&l=${lang}&t=${time}&engine=${engine}`)
.then((res) => res.json())
.then((data: suggestionsResponse) => {
.then((data: SuggestionsResponse) => {
try {
const suggestionToUpdate: suggestionItem[] = data.suggestions;
const suggestionToUpdate: SuggestionItem[] = data.suggestions;
if (data.time > lastRequestTimeRef.current) {
cleanSuggestion("NAVIGATION", "QUERY");
lastRequestTimeRef.current = data.time;
@ -73,8 +56,8 @@ export default function OneSearch() {
});
}, [lastInput]);
function updateSuggestion(data: suggestionItem[]) {
setFinalSuggetsion((cur: suggestionItem[]) => {
function updateSuggestion(data: SuggestionItem[]) {
setFinalSuggestion((cur: SuggestionItem[]) => {
const types: string[] = [];
for (const sug of data) {
if (!types.includes(sug.type)) types.push(sug.type);
@ -91,7 +74,7 @@ export default function OneSearch() {
}
function cleanSuggestion(...types: string[]) {
setFinalSuggetsion((suggestion: suggestionItem[]) => {
setFinalSuggestion((suggestion: SuggestionItem[]) => {
return suggestion.filter((item) => {
return !types.includes(item.type);
});
@ -99,52 +82,7 @@ export default function OneSearch() {
}
useEffect(() => {
if (embeddingLayer !== null) return;
const embedding_file = "/model/token_embeddings.bin";
(async function () {
const result = await fetch(embedding_file);
const arrBuf = await result.arrayBuffer();
const embeddingDict = getEmbeddingLayer(arrBuf);
setEmbeddingLayer(embeddingDict);
await loadModel("/model/NLU.onnx");
// if (!modelLoaded) {
// console.error("NLU model was not correctly loaded.")
// }
})();
}, []);
useEffect(() => {
if (tokenizer !== null) return;
(async function () {
await loadTokenizer();
})();
}, []);
async function loadModel(modelPath: string) {
ort.env.wasm.wasmPaths = "/onnx/";
const session = await ort.InferenceSession.create(modelPath);
setNLUsession(session);
}
async function loadTokenizer() {
const vocab = await loadVocab();
const tokenizer = new BPETokenizer(vocab);
setTokenizer(tokenizer);
}
async function getNLUResult(query: string) {
if (embeddingLayer === null || NLUsession === null || tokenizer == null) return;
const tokenIds = await tokenize(bytesToUnicode(query), tokenizer);
const embeddings = getEmbedding(tokenIds, embeddingLayer, 64);
const inputTensor = new ort.Tensor("float32", embeddings, [1, 64, 96]);
const feeds = { input: inputTensor };
const results = await NLUsession.run(feeds);
return results;
}
useEffect(() => {
cleanSuggestion("default-link", "default", "text", "link");
cleanSuggestion("default-link", "default", "text", "link", "inpage-link");
if (validLink(query)) {
updateSuggestion([
{
@ -164,105 +102,24 @@ export default function OneSearch() {
}
]);
}
if (keywordSuggestion(query) !== null) {
updateSuggestion([keywordSuggestion(query)!]);
}
(async function () {
const result = await getNLUResult(query);
if (result === undefined) return;
const rawData = result.output.data;
const data: number[] = [];
for (let i=0;i<rawData.length;i++){
data.push(rawData[i] as number);
}
console.log(data, energyScore(data));
})();
}, [lastInput, engineName]);
return (
<SuggestionBox>
{suggestion.map((s, i) => {
if (s.suggestion.trim() === "") return;
if (s.type === "default") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}&nbsp;
<span className="text-zinc-700 dark:text-zinc-400 text-sm">
{t("search.search-help-text", { engine: engineName })}
</span>
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (s.type === "QUERY") {
return (
<PlainSearch key={i} query={s.suggestion} selected={i == selected}>
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainSearch>
);
} else if (
s.type === "NAVIGATION" ||
s.type === "default-link" ||
s.type === "link"
) {
return (
<LinkSuggestion key={i} query={s.suggestion} selected={i == selected}>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
} else if (s.type === "text") {
return (
<PlainText key={i} selected={i == selected}>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
<p>{s.suggestion}</p>
{devMode && (
<span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</PlainText>
);
} else if (s.type === "inpage-link") {
return (
<LinkSuggestion
{suggestion.map((s, i) => (
<SuggestionComponent
key={i}
query={s.suggestion}
selected={i == selected}
inPage={true}
>
{s.prompt && (
<span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>
)}
{s.suggestion}
{devMode && (
<span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2">
{s.relevance}
</span>
)}
</LinkSuggestion>
);
}
})}
s={s}
i={i}
selected={selected}
devMode={devMode}
engineName={engineName}
t={t}
/>
))}
</SuggestionBox>
);
}

6
global.d.ts vendored
View File

@ -1,4 +1,4 @@
import { Suggestion } from "search-engine-autocomplete";
import React from "react";
interface settingsType extends object {
version: number;
@ -12,14 +12,14 @@ interface settingsType extends object {
};
}
interface suggestionsResponse extends object {
interface SuggestionsResponse extends object {
suggestions: Suggestion[];
query: string;
verbatimRelevance: number;
time: number;
}
type suggestionItem = {
interface SuggestionItem {
suggestion: string;
type: string;
relativeRelevance?: number;

21
i18n/en.json
View File

@ -33,7 +33,26 @@
"about": {
"title": "About sparkast",
"license": {
"view": "→ view"
"title": "License",
"view": "→ view",
"text": "GPL 3.0"
},
"presented-by": "Presented By",
"backend-api-version": "Backend API Version",
"client-version": "Client API Version",
"oss": {
"title": "OSS Statement",
"view": "→ View"
}
},
"license": {
"title": "License - sparkast",
"desc": "sparkast is open source software. It is licensed under the GPL 3.0 license. You can view the full license text below.",
"page-title": "License"
},
"oss_license": {
"title": "OSS Licenses - sparkast",
"desc": "This birth of this project would not have been possible without the following open source software. This page shows the licenses of the open source software used in sparkast.",
"page-title": "OSS Licenses"
}
}

23
i18n/zh.json
View File

@ -32,8 +32,27 @@
},
"about": {
"title": "关于 sparkast",
"license": {
"oss": {
"title": "开源软件声明",
"view": "→ 查看"
}
},
"license": {
"title": "许可证",
"text": "GPL 3.0",
"view": "→ 查看"
},
"presented-by": "出品",
"backend-api-version": "后端 API 版本",
"client-version": "客户端 API 版本"
},
"license": {
"title": "开源许可证 - sparkast",
"page-title": "开源许可证",
"desc": "sparkast 是开源软件。它根据 GPL 3.0 许可证获得许可。您可以在下面查看完整的许可文本。"
},
"oss_license": {
"title": "开源软件声明 - sparkast",
"page-title": "开源软件声明",
"desc": "本项目的诞生离不开开源软件。此页面显示 sparkast 中使用的开源软件的许可证。"
}
}

6
index.html
View File

@ -1,10 +1,12 @@
<!doctype html>
<html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" href="/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>SparkHome</title>
<meta property="og:title" content="sparkast">
<meta property="og:description" content="Your ultimate productivity tool that gives you the answer to all within a single search box.">
<title>sparkast</title>
</head>
<body>
<div id="root"></div>

6
lib/onesearch/handleEnter.ts
View File

@ -1,11 +1,11 @@
import { settingsType, suggestionItem } from "global";
import { settingsType, SuggestionItem } from "global";
import copyToClipboard from "lib/copy";
import { normalizeURL } from "lib/normalizeURL";
import search from "lib/search";
export default function (
index: number,
suggestion: suggestionItem[],
suggestion: SuggestionItem[],
_query: string,
settings: settingsType,
searchBoxRef: React.RefObject<HTMLInputElement>
@ -23,6 +23,6 @@ export default function (
} else if (selected.type === "link") {
window.open(normalizeURL(selected.suggestion));
} else if (selected.type === "inpage-link") {
location.href = normalizeURL(selected.suggestion);
location.href = normalizeURL(selected.suggestion, false);
}
}

6
lib/onesearch/keywordSuggestion.ts
View File

@ -1,4 +1,4 @@
import { suggestionItem } from "global";
import { SuggestionItem } from "global";
interface keywordLinkDict {
[key: string]: string;
@ -15,7 +15,7 @@ const dict_cn: keywordLinkDict = {
export function keywordSuggestion(query: string) {
for (const keyword in dict_cn) {
if (query.includes(keyword)) {
const result: suggestionItem = {
const result: SuggestionItem = {
type: "inpage-link",
suggestion: dict_cn[keyword],
prompt: keyword,
@ -26,7 +26,7 @@ export function keywordSuggestion(query: string) {
}
for (const keyword in dict_en) {
if (query.includes(keyword)) {
const result: suggestionItem = {
const result: SuggestionItem = {
type: "inpage-link",
suggestion: dict_en[keyword],
prompt: keyword,

4
lib/state/suggestion.ts
View File

@ -1,6 +1,6 @@
import { suggestionItem } from "global";
import { SuggestionItem } from "global";
import { atom } from "jotai";
const suggestionAtom = atom([] as suggestionItem[]);
const suggestionAtom = atom([] as SuggestionItem[]);
export { suggestionAtom };

2
lib/url/tldList.ts
View File

@ -1,4 +1,4 @@
import TLDtxt from "./tlds.txt?raw";
import TLDtxt from "./tlds.txt";
export function getTLD() {
return TLDtxt.split("\r\n").filter((line) => line[0] !== "#");

3
lib/url/tlds.txt
View File

@ -1,4 +1,4 @@
# Version 2024071300, Last Updated Sat Jul 13 07:07:01 2024 UTC
# Version 2025011900, Last Updated Sun Jan 19 07:07:01 2025 UTC
AAA
AARP
ABB
@ -297,7 +297,6 @@ CY
CYMRU
CYOU
CZ
DABUR
DAD
DANCE
DATA

30
lib/url/validLink.ts
View File

@ -1,41 +1,33 @@
import { toASCII } from "tr46";
import { getTLD } from "./tldList";
console.log(getTLD());
export default function validLink(link: string) {
let finalURL;
try {
const url = new URL(link);
finalURL = url;
new URL(link);
return true;
} catch (error) {
// if the URL is invalid, try to add the protocol
try {
const urlWithHTTP = new URL("http://" + link);
finalURL = urlWithHTTP;
finalURL = new URL("http://" + link);
} catch (error) {
return false;
}
}
if (finalURL.host.endsWith(".")) return false;
if (
validTLD(finalURL.host) ||
return validTLD(finalURL.host) ||
isValidIPv6(link.slice(1, finalURL.host.length - 1)) ||
isValidIPv4(link)
) {
return true;
}
return false;
isValidIPv4(link);
}
export function validTLD(domain: string): boolean {
if (!domain.includes(".")) return false;
const tld = toASCII(domain.split(".").reverse()[0]);
const tldList = getTLD();
if (tldList.includes(tld.toUpperCase())) {
return true;
} else {
return false;
}
return !!tldList.includes(tld.toUpperCase());
}
export function isValidIPv6(ip: string): boolean {
@ -69,10 +61,8 @@ export function isValidIPv6(ip: string): boolean {
return false;
}
}
if (doubleColonCount === 0 && groups !== 8) {
return false;
}
return true;
return !(doubleColonCount === 0 && groups !== 8);
}
export function isValidIPv4(ip: string): boolean {

5
lib/version.ts
View File

@ -1,8 +1,5 @@
import * as pjson from "package.json";
export default function getVersion() {
return pjson.version;
}
export const clientVersion = pjson.version;
export const clientNLUVersion = 4;
export const apiVersion = 1;

2
package.json
View File

@ -34,6 +34,7 @@
"onnxruntime-web": "^1.20.0-dev.20240925-a47254eaef",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-helmet": "^6.1.0",
"react-i18next": "^14.1.2",
"react-router": "^6.23.1",
"react-router-dom": "^6.23.1",
@ -45,6 +46,7 @@
"devDependencies": {
"@types/react": "^18.3.3",
"@types/react-dom": "^18.3.0",
"@types/react-helmet": "^6.1.11",
"@types/valid-url": "^1.0.7",
"@typescript-eslint/eslint-plugin": "^7.13.1",
"@typescript-eslint/parser": "^7.13.1",

66
pages/about/index.tsx
View File

@ -1,8 +1,38 @@
import useDarkMode from "lib/darkModeHook";
import getVersion, { apiVersion, clientNLUVersion } from "lib/version";
import { apiVersion, clientVersion } from "lib/version";
import AboutLayout from "./layout";
import { useTranslation } from "react-i18next";
function License() {
const { t } = useTranslation();
return (
<>
<p className="flex items-center my-4">
<span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t("about.oss.title")}
</span>
<span
className="relative font-bold px-2 py-1 rounded-md text-nowrap underline
bg-green-600 text-white"
>
<a href="/about/oss-licenses">{t("about.oss.view")}</a>
</span>
</p>
<p className="flex items-center my-4">
<span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t("about.license.title")}
</span>
<span
className="relative font-bold px-2 py-1 rounded-md text-nowrap underline
bg-red-500 text-white"
>
<a href="/about/license">{t("about.license.text")}</a>
</span>
</p>
</>
);
}
export default function AboutPage() {
const darkMode = useDarkMode();
const { t } = useTranslation();
@ -26,47 +56,41 @@ export default function AboutPage() {
</div>
</div>
<Version title="Frontend Version" version={getVersion()} versionClass="bg-red-500" />
<Version
title="Browser NLU Model Version"
version={"Build " + clientNLUVersion}
title={t("about.backend-api-version")}
version={"/api/v" + apiVersion}
versionClass="bg-purple-500"
/>
<Version
title="Backend API Version"
version={"/api/v" + apiVersion}
title={t("about.client-version")}
version={clientVersion}
versionClass="bg-orange-500"
/>
<p className="flex items-center my-3">
<span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">License</span>
<span
className="relative px-2 py-1 text-sm font-bold rounded-md text-nowrap underline
bg-green-600 text-white"
>
<a href="/about/license">{t("about.license.view")}</a>
</span>
</p>
<p className="relative font-bold text-2xl mt-12">Presented By</p>
<License />
<p className="relative font-bold text-2xl mt-12">{t("about.presented-by")}</p>
{!darkMode && (
<img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" />
)}
{darkMode && (
<img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-56 mt-6" />
<img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-64 mt-6" />
)}
</AboutLayout>
);
}
function Version(props: { title: string; version: string; versionClass?: string }) {
document.title = "About SparkHome";
const { t } = useTranslation();
return (
<p className="flex items-center my-3">
<span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">{t(props.title)}</span>
<p className="flex items-center my-4">
<span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]">
{t(props.title)}
</span>
<span
className={
"relative px-2 py-1 text-sm font-bold rounded-md text-nowrap text-white " +
"relative px-2 py-1 font-bold rounded-md text-nowrap text-white " +
props.versionClass || ""
}
>

8
pages/about/layout.tsx
View File

@ -1,6 +1,14 @@
import React from "react";
import { useTranslation } from "react-i18next";
import { Helmet } from "react-helmet";
export default function AboutLayout({ children }: { children: React.ReactNode }) {
const { t } = useTranslation();
return (
<div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]">
<Helmet>
<title>{t('about.title')}</title>
</Helmet>
<main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12 px-3 md:px-0"

11
pages/about/license/index.tsx → pages/about/license/OSS.tsx
View File

@ -1,13 +1,20 @@
import LICENSE from "lib/license.txt?raw";
import { Helmet } from "react-helmet";
import { useTranslation } from "react-i18next";
export default function LicensePage() {
export default function OSSLicensesPage() {
const { t } = useTranslation();
return (
<div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden">
<Helmet>
<title>{t("oss_license.title")}</title>
</Helmet>
<main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12"
>
<h1 className="text-4xl font-bold mb-6">LICENSE</h1>
<h1 className="text-4xl font-bold mb-6">{t('oss_license.page-title')}</h1>
<p className="text-lg mb-8">{t('oss_license.desc')}</p>
<div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div>
</main>
</div>

22
pages/about/license/ThisProject.tsx Normal file
View File

@ -0,0 +1,22 @@
import LICENSE from "LICENSE?raw";
import { useTranslation } from "react-i18next";
import { Helmet } from "react-helmet";
export default function ThisProjectLicensePage() {
const { t } = useTranslation();
return (
<div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden">
<Helmet>
<title>{t("license.title")}</title>
</Helmet>
<main
className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4
pt-12"
>
<h1 className="text-4xl font-bold mb-6">{t('license.page-title')}</h1>
<p className="text-lg mb-8">{t('license.desc')}</p>
<div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div>
</main>
</div>
);
}

1
pages/index.tsx
View File

@ -13,6 +13,7 @@ export default function Homepage() {
return (
<div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]">
<title>sparkast</title>
<Background />
<EngineSelector

17
src/app.tsx
View File

@ -3,7 +3,8 @@ import { createBrowserRouter, RouterProvider } from "react-router-dom";
import "./i18n";
import Homepage from "pages";
import AboutPage from "pages/about";
import LicensePage from "pages/about/license";
import OSSLicensesPage from "pages/about/license/OSS";
import ThisProjectLicensePage from "../pages/about/license/ThisProject.tsx";
const router = createBrowserRouter([
{
@ -11,14 +12,16 @@ const router = createBrowserRouter([
element: <Homepage />
},
{
path: "about",
path: "/about",
element: <AboutPage />,
children: [
},
{
path: "license",
element: <LicensePage />
}
]
path: "/about/oss-licenses",
element: <OSSLicensesPage />
},
{
path: "/about/license",
element: <ThisProjectLicensePage />
}
]);

6
test/bytesToUnicode.test.ts
View File

@ -1,6 +0,0 @@
import { expect, test } from "bun:test";
import bytesToUnicodes from "../lib/nlp/tokenize/bytesToUnicode";
test("bytesToUnicode: test", () => {
expect(bytesToUnicodes("Hello 你好")).toEqual("HelloĠä½łå¥½");
});

0
test/getEmbeddings.test.ts
View File

6
test/unicodeToBytes.test.ts
View File

@ -1,6 +0,0 @@
import { expect, test } from "bun:test";
import unicodeToBytes from "../lib/nlp/tokenize/unicodeToBytes";
test("unicodeToBytes: test", () => {
expect(unicodeToBytes("HelloĠä½łå¥½")).toEqual("Hello 你好");
});

1
test/validLink.test.ts
View File

@ -60,6 +60,7 @@ describe("Check if the given TLD exist and assigned.", () => {
expect(validTLD("example.foo")).toBe(true);
expect(validTLD("example.bar")).toBe(true);
expect(validTLD("example.zip")).toBe(true);
expect(validTLD("xn--s8w913fdga.chn.moe")).toBe(true);
});
test("Exist but not assigned TLD", () => {
expect(validTLD("example.active")).toBe(false);