diff --git a/backend/nlp/.ipynb_checkpoints/Text Classification-checkpoint.ipynb b/backend/nlp/.ipynb_checkpoints/Text Classification-checkpoint.ipynb deleted file mode 100644 index 8bf44f5..0000000 --- a/backend/nlp/.ipynb_checkpoints/Text Classification-checkpoint.ipynb +++ /dev/null @@ -1,33 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/backend/nlp/Text Classification.ipynb b/backend/nlp/Text Classification.ipynb deleted file mode 100644 index 9820743..0000000 --- a/backend/nlp/Text Classification.ipynb +++ /dev/null @@ -1,87 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 14, - "id": "0fbcb787-a093-4c31-96a0-22f65a3ee25b", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0mRequirement already satisfied: huggingface_hub in /opt/homebrew/lib/python3.11/site-packages (0.20.3)\n", - "Requirement already satisfied: filelock in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (3.13.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2023.10.0)\n", - "Requirement already satisfied: requests in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (2.31.0)\n", - "Requirement already satisfied: tqdm>=4.42.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.66.2)\n", - "Requirement already satisfied: pyyaml>=5.1 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (6.0.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (4.8.0)\n", - "Requirement already satisfied: packaging>=20.9 in /opt/homebrew/lib/python3.11/site-packages (from huggingface_hub) (23.2)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2.1.0)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /opt/homebrew/lib/python3.11/site-packages (from requests->huggingface_hub) (2023.11.17)\n", - "\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/numpy-1.26.4.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/certifi-2024.2.2.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m\u001b[33mWARNING: Skipping /opt/homebrew/lib/python3.11/site-packages/packaging-24.0.dist-info due to invalid metadata entry 'name'\u001b[0m\u001b[33m\n", - "\u001b[0m" - ] - } - ], - "source": [ - "!pip install huggingface_hub" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e82bbe25-4b14-4aca-8761-bad2a63fccd2", - "metadata": {}, - "outputs": [ - { - "ename": "ModuleNotFoundError", - "evalue": "No module named 'huggingface_hub'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mhuggingface_hub\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m notebook_login\n", - "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'huggingface_hub'" - ] - } - ], - "source": [ - "from huggingface_hub import notebook_login" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/bun.lockb b/bun.lockb index 823f6fb..56264ed 100755 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/components/onesearch/SuggestionItem.tsx b/components/onesearch/SuggestionItem.tsx new file mode 100644 index 0000000..25b507e --- /dev/null +++ b/components/onesearch/SuggestionItem.tsx @@ -0,0 +1,87 @@ +import { SuggestionItem } from "global"; +import PlainSearch from "./plainSearch"; +import LinkSuggestion from "./link"; +import PlainText from "./plainText"; + +export default function SuggestionComponent({ + s, + i, + selected, + devMode, + engineName, + t +}: { + s: SuggestionItem; + i: number; + selected: number; + devMode: boolean; + engineName: string; + t: any; +}) { + if (s.suggestion.trim() === "") return null; + + if (s.type === "default") { + return ( + + {s.suggestion}  + + {t("search.search-help-text", { engine: engineName })} + + {devMode && ( + + {s.relevance} + + )} + + ); + } else if (s.type === "QUERY") { + return ( + + {s.suggestion} + {devMode && ( + + {s.relevance} + + )} + + ); + } else if (s.type === "NAVIGATION" || s.type === "default-link" || s.type === "link") { + return ( + + {s.prompt && {s.prompt}} + {s.suggestion} + {devMode && ( + + {s.relevance} + + )} + + ); + } else if (s.type === "text") { + return ( + + {s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>} + <p>{s.suggestion}</p> + {devMode && ( + <span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> + {s.relevance} + </span> + )} + </PlainText> + ); + } else if (s.type === "inpage-link") { + return ( + <LinkSuggestion key={i} query={s.suggestion} selected={i == selected} inPage={true}> + {s.prompt && <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span>} + {s.suggestion} + {devMode && ( + <span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> + {s.relevance} + </span> + )} + </LinkSuggestion> + ); + } + + return null; +} \ No newline at end of file diff --git a/components/onesearch/handleEnter.ts b/components/onesearch/handleEnter.ts index aaa799e..ec61000 100644 --- a/components/onesearch/handleEnter.ts +++ b/components/onesearch/handleEnter.ts @@ -1,11 +1,11 @@ -import { settingsType, suggestionItem } from "global"; +import { settingsType, SuggestionItem } from "global"; import copyToClipboard from "lib/copy"; import { normalizeURL } from "lib/normalizeURL"; import search from "lib/search"; export default function ( index: number, - suggestion: suggestionItem[], + suggestion: SuggestionItem[], _query: string, settings: settingsType, searchBoxRef: React.RefObject<HTMLInputElement> diff --git a/components/onesearch/handleNLUResult.ts b/components/onesearch/handleNLUResult.ts index fa210dd..c90543b 100644 --- a/components/onesearch/handleNLUResult.ts +++ b/components/onesearch/handleNLUResult.ts @@ -1,10 +1,10 @@ -import { suggestionItem } from "global"; +import { SuggestionItem } from "global"; import { findClosestDateIndex } from "lib/weather/getCurrentWeather"; import { getLocationNative } from "lib/weather/getLocation"; import { getWeather } from "lib/weather/getWeather"; import { WMOCodeTable } from "lib/weather/wmocode"; -type UpdateSuggestionFunction = (data: suggestionItem[]) => void; +type UpdateSuggestionFunction = (data: SuggestionItem[]) => void; export function handleNLUResult(result: any, updateSuggestion: UpdateSuggestionFunction) { if (result.intent == "weather.summary") { diff --git a/components/onesearch/onesearch.tsx b/components/onesearch/onesearch.tsx index 6f85277..cb314da 100644 --- a/components/onesearch/onesearch.tsx +++ b/components/onesearch/onesearch.tsx @@ -1,39 +1,22 @@ -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef } from "react"; import SuggestionBox from "./suggestionBox"; import { queryAtom } from "lib/state/query"; -import { suggestionItem, suggestionsResponse } from "global"; +import { SuggestionItem, SuggestionsResponse } from "global"; import getSearchEngineName from "lib/onesearch/getSearchEngineName"; -import PlainSearch from "./plainSearch"; import { suggestionAtom } from "lib/state/suggestion"; import validLink from "lib/url/validLink"; -import LinkSuggestion from "./link"; import { selectedSuggestionAtom } from "lib/state/suggestionSelection"; import { settingsAtom } from "lib/state/settings"; -import PlainText from "./plainText"; import { sendError } from "lib/feedback/sendError"; -import { handleNLUResult } from "./handleNLUResult"; -import * as ort from "onnxruntime-web"; import { useAtom, useAtomValue } from "jotai"; import i18next from "i18next"; import { useTranslation } from "react-i18next"; import { keywordSuggestion } from "lib/onesearch/keywordSuggestion"; -import tokenize from "lib/nlp/tokenize/tokenizer"; -import { getEmbedding, getEmbeddingLayer } from "lib/nlp/getEmbedding"; -import { loadVocab } from "lib/nlp/tokenize/loadVocab"; -import BPETokenizer from "lib/nlp/tokenize/BPEtokenizer"; -import energyScore from "lib/nlp/energyScore"; -import bytesToUnicode from "lib/nlp/tokenize/bytesToUnicode"; -import { searchboxLastInputAtom } from "lib/state/searchboxLastInput"; - -interface EmbeddingLayer { - [key: number]: Float32Array<ArrayBufferLike>; -} +import { searchboxLastInputAtom } from "lib/state/searchboxLastInput" +import SuggestionComponent from "./SuggestionItem.tsx"; export default function OneSearch() { - const [suggestion, setFinalSuggetsion] = useAtom(suggestionAtom); - const [embeddingLayer, setEmbeddingLayer] = useState<EmbeddingLayer | null>(null); - const [NLUsession, setNLUsession] = useState<ort.InferenceSession | null>(null); - const [tokenizer, setTokenizer] = useState<BPETokenizer | null>(null); + const [suggestion, setFinalSuggestion] = useAtom(suggestionAtom); const lastInput = useAtomValue(searchboxLastInputAtom); const lastRequestTimeRef = useRef(0); const selected = useAtomValue(selectedSuggestionAtom); @@ -53,9 +36,9 @@ export default function OneSearch() { } fetch(`/api/v1/suggestion?q=${query}&l=${lang}&t=${time}&engine=${engine}`) .then((res) => res.json()) - .then((data: suggestionsResponse) => { + .then((data: SuggestionsResponse) => { try { - const suggestionToUpdate: suggestionItem[] = data.suggestions; + const suggestionToUpdate: SuggestionItem[] = data.suggestions; if (data.time > lastRequestTimeRef.current) { cleanSuggestion("NAVIGATION", "QUERY"); lastRequestTimeRef.current = data.time; @@ -73,8 +56,8 @@ export default function OneSearch() { }); }, [lastInput]); - function updateSuggestion(data: suggestionItem[]) { - setFinalSuggetsion((cur: suggestionItem[]) => { + function updateSuggestion(data: SuggestionItem[]) { + setFinalSuggestion((cur: SuggestionItem[]) => { const types: string[] = []; for (const sug of data) { if (!types.includes(sug.type)) types.push(sug.type); @@ -91,7 +74,7 @@ export default function OneSearch() { } function cleanSuggestion(...types: string[]) { - setFinalSuggetsion((suggestion: suggestionItem[]) => { + setFinalSuggestion((suggestion: SuggestionItem[]) => { return suggestion.filter((item) => { return !types.includes(item.type); }); @@ -99,52 +82,7 @@ export default function OneSearch() { } useEffect(() => { - if (embeddingLayer !== null) return; - const embedding_file = "/model/token_embeddings.bin"; - (async function () { - const result = await fetch(embedding_file); - const arrBuf = await result.arrayBuffer(); - const embeddingDict = getEmbeddingLayer(arrBuf); - setEmbeddingLayer(embeddingDict); - - await loadModel("/model/NLU.onnx"); - // if (!modelLoaded) { - // console.error("NLU model was not correctly loaded.") - // } - })(); - }, []); - - useEffect(() => { - if (tokenizer !== null) return; - (async function () { - await loadTokenizer(); - })(); - }, []); - - async function loadModel(modelPath: string) { - ort.env.wasm.wasmPaths = "/onnx/"; - const session = await ort.InferenceSession.create(modelPath); - setNLUsession(session); - } - - async function loadTokenizer() { - const vocab = await loadVocab(); - const tokenizer = new BPETokenizer(vocab); - setTokenizer(tokenizer); - } - - async function getNLUResult(query: string) { - if (embeddingLayer === null || NLUsession === null || tokenizer == null) return; - const tokenIds = await tokenize(bytesToUnicode(query), tokenizer); - const embeddings = getEmbedding(tokenIds, embeddingLayer, 64); - const inputTensor = new ort.Tensor("float32", embeddings, [1, 64, 96]); - const feeds = { input: inputTensor }; - const results = await NLUsession.run(feeds); - return results; - } - - useEffect(() => { - cleanSuggestion("default-link", "default", "text", "link"); + cleanSuggestion("default-link", "default", "text", "link", "inpage-link"); if (validLink(query)) { updateSuggestion([ { @@ -164,105 +102,24 @@ export default function OneSearch() { } ]); } - if (keywordSuggestion(query) !== null) { updateSuggestion([keywordSuggestion(query)!]); } - - (async function () { - const result = await getNLUResult(query); - if (result === undefined) return; - const rawData = result.output.data; - const data: number[] = []; - for (let i=0;i<rawData.length;i++){ - data.push(rawData[i] as number); - } - console.log(data, energyScore(data)); - })(); }, [lastInput, engineName]); return ( <SuggestionBox> - {suggestion.map((s, i) => { - if (s.suggestion.trim() === "") return; - if (s.type === "default") { - return ( - <PlainSearch key={i} query={s.suggestion} selected={i == selected}> - {s.suggestion}&nbsp; - <span className="text-zinc-700 dark:text-zinc-400 text-sm"> - {t("search.search-help-text", { engine: engineName })} - </span> - {devMode && ( - <span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> - {s.relevance} - </span> - )} - </PlainSearch> - ); - } else if (s.type === "QUERY") { - return ( - <PlainSearch key={i} query={s.suggestion} selected={i == selected}> - {s.suggestion} - {devMode && ( - <span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> - {s.relevance} - </span> - )} - </PlainSearch> - ); - } else if ( - s.type === "NAVIGATION" || - s.type === "default-link" || - s.type === "link" - ) { - return ( - <LinkSuggestion key={i} query={s.suggestion} selected={i == selected}> - {s.prompt && ( - <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span> - )} - {s.suggestion} - {devMode && ( - <span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> - {s.relevance} - </span> - )} - </LinkSuggestion> - ); - } else if (s.type === "text") { - return ( - <PlainText key={i} selected={i == selected}> - {s.prompt && ( - <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span> - )} - <p>{s.suggestion}</p> - {devMode && ( - <span className="bottom-0 absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> - {s.relevance} - </span> - )} - </PlainText> - ); - } else if (s.type === "inpage-link") { - return ( - <LinkSuggestion - key={i} - query={s.suggestion} - selected={i == selected} - inPage={true} - > - {s.prompt && ( - <span className="text-zinc-700 dark:text-zinc-400">{s.prompt}</span> - )} - {s.suggestion} - {devMode && ( - <span className="absolute text-zinc-700 dark:text-zinc-400 text-sm leading-10 h-10 right-2"> - {s.relevance} - </span> - )} - </LinkSuggestion> - ); - } - })} + {suggestion.map((s, i) => ( + <SuggestionComponent + key={i} + s={s} + i={i} + selected={selected} + devMode={devMode} + engineName={engineName} + t={t} + /> + ))} </SuggestionBox> ); } diff --git a/global.d.ts b/global.d.ts index 115aa9a..6a5ccf5 100644 --- a/global.d.ts +++ b/global.d.ts @@ -1,4 +1,4 @@ -import { Suggestion } from "search-engine-autocomplete"; +import React from "react"; interface settingsType extends object { version: number; @@ -12,14 +12,14 @@ interface settingsType extends object { }; } -interface suggestionsResponse extends object { +interface SuggestionsResponse extends object { suggestions: Suggestion[]; query: string; verbatimRelevance: number; time: number; } -type suggestionItem = { +interface SuggestionItem { suggestion: string; type: string; relativeRelevance?: number; diff --git a/i18n/en.json b/i18n/en.json index 2206878..0362f21 100755 --- a/i18n/en.json +++ b/i18n/en.json @@ -33,7 +33,26 @@ "about": { "title": "About sparkast", "license": { - "view": "→ view" + "title": "License", + "view": "→ view", + "text": "GPL 3.0" + }, + "presented-by": "Presented By", + "backend-api-version": "Backend API Version", + "client-version": "Client API Version", + "oss": { + "title": "OSS Statement", + "view": "→ View" } + }, + "license": { + "title": "License - sparkast", + "desc": "sparkast is open source software. It is licensed under the GPL 3.0 license. You can view the full license text below.", + "page-title": "License" + }, + "oss_license": { + "title": "OSS Licenses - sparkast", + "desc": "This birth of this project would not have been possible without the following open source software. This page shows the licenses of the open source software used in sparkast.", + "page-title": "OSS Licenses" } } diff --git a/i18n/zh.json b/i18n/zh.json index 19c32d7..6f75279 100644 --- a/i18n/zh.json +++ b/i18n/zh.json @@ -1,39 +1,58 @@ { - "404": { - "title": "页面未找到" - }, - "search": { - "placeholder": "搜索或输入网址", - "engine-aria": "切换搜索引擎", - "engine": { - "google": "谷歌", - "baidu": "百度", - "bing": "必应", - "duckduckgo": "DuckDuckGo", - "yandex": "Yandex", - "yahoo": "雅虎", - "ecosia": "Ecosia" - }, - "search-help-text": "用 {engine} 搜索" - }, - "tools": { - "base64": { - "title": "Base64 工具", - "decode": "解码", - "encode": "编码", - "result": "结果： ", - "copy": "复制", - "copied": "已复制" - } - }, - "notfound": { - "desc": "请检查网址是否出错。 <br/>如果你从星火主页跳转到这里，<br/> 请 <a style=\"text-decoration:underline;\" href=\"mailto:contact@alikia2x.com\">联系我们</a>", - "title": "网页不存在" - }, - "about": { - "title": "关于 sparkast", - "license": { - "view": "→ 查看" - } - } + "404": { + "title": "页面未找到" + }, + "search": { + "placeholder": "搜索或输入网址", + "engine-aria": "切换搜索引擎", + "engine": { + "google": "谷歌", + "baidu": "百度", + "bing": "必应", + "duckduckgo": "DuckDuckGo", + "yandex": "Yandex", + "yahoo": "雅虎", + "ecosia": "Ecosia" + }, + "search-help-text": "用 {engine} 搜索" + }, + "tools": { + "base64": { + "title": "Base64 工具", + "decode": "解码", + "encode": "编码", + "result": "结果： ", + "copy": "复制", + "copied": "已复制" + } + }, + "notfound": { + "desc": "请检查网址是否出错。 <br/>如果你从星火主页跳转到这里，<br/> 请 <a style=\"text-decoration:underline;\" href=\"mailto:contact@alikia2x.com\">联系我们</a>", + "title": "网页不存在" + }, + "about": { + "title": "关于 sparkast", + "oss": { + "title": "开源软件声明", + "view": "→ 查看" + }, + "license": { + "title": "许可证", + "text": "GPL 3.0", + "view": "→ 查看" + }, + "presented-by": "出品", + "backend-api-version": "后端 API 版本", + "client-version": "客户端 API 版本" + }, + "license": { + "title": "开源许可证 - sparkast", + "page-title": "开源许可证", + "desc": "sparkast 是开源软件。它根据 GPL 3.0 许可证获得许可。您可以在下面查看完整的许可文本。" + }, + "oss_license": { + "title": "开源软件声明 - sparkast", + "page-title": "开源软件声明", + "desc": "本项目的诞生离不开开源软件。此页面显示 sparkast 中使用的开源软件的许可证。" + } } diff --git a/index.html b/index.html index af9602f..80f1bd1 100644 --- a/index.html +++ b/index.html @@ -1,10 +1,12 @@ <!doctype html> -<html> +<html lang="en"> <head> <meta charset="UTF-8" /> <link rel="icon" href="/favicon.ico" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <title>SparkHome</title> + <meta property="og:title" content="sparkast"> + <meta property="og:description" content="Your ultimate productivity tool that gives you the answer to all within a single search box."> + <title>sparkast</title> </head> <body> <div id="root"></div> diff --git a/lib/onesearch/handleEnter.ts b/lib/onesearch/handleEnter.ts index 1dbc8e4..949d24a 100644 --- a/lib/onesearch/handleEnter.ts +++ b/lib/onesearch/handleEnter.ts @@ -1,11 +1,11 @@ -import { settingsType, suggestionItem } from "global"; +import { settingsType, SuggestionItem } from "global"; import copyToClipboard from "lib/copy"; import { normalizeURL } from "lib/normalizeURL"; import search from "lib/search"; export default function ( index: number, - suggestion: suggestionItem[], + suggestion: SuggestionItem[], _query: string, settings: settingsType, searchBoxRef: React.RefObject<HTMLInputElement> @@ -23,6 +23,6 @@ export default function ( } else if (selected.type === "link") { window.open(normalizeURL(selected.suggestion)); } else if (selected.type === "inpage-link") { - location.href = normalizeURL(selected.suggestion); + location.href = normalizeURL(selected.suggestion, false); } } diff --git a/lib/onesearch/keywordSuggestion.ts b/lib/onesearch/keywordSuggestion.ts index c13cfc0..7c11e38 100644 --- a/lib/onesearch/keywordSuggestion.ts +++ b/lib/onesearch/keywordSuggestion.ts @@ -1,4 +1,4 @@ -import { suggestionItem } from "global"; +import { SuggestionItem } from "global"; interface keywordLinkDict { [key: string]: string; @@ -15,7 +15,7 @@ const dict_cn: keywordLinkDict = { export function keywordSuggestion(query: string) { for (const keyword in dict_cn) { if (query.includes(keyword)) { - const result: suggestionItem = { + const result: SuggestionItem = { type: "inpage-link", suggestion: dict_cn[keyword], prompt: keyword, @@ -26,7 +26,7 @@ export function keywordSuggestion(query: string) { } for (const keyword in dict_en) { if (query.includes(keyword)) { - const result: suggestionItem = { + const result: SuggestionItem = { type: "inpage-link", suggestion: dict_en[keyword], prompt: keyword, diff --git a/lib/state/suggestion.ts b/lib/state/suggestion.ts index df90be4..7fccb90 100644 --- a/lib/state/suggestion.ts +++ b/lib/state/suggestion.ts @@ -1,6 +1,6 @@ -import { suggestionItem } from "global"; +import { SuggestionItem } from "global"; import { atom } from "jotai"; -const suggestionAtom = atom([] as suggestionItem[]); +const suggestionAtom = atom([] as SuggestionItem[]); export { suggestionAtom }; diff --git a/lib/url/tldList.ts b/lib/url/tldList.ts index 950ae99..8fc1f8d 100644 --- a/lib/url/tldList.ts +++ b/lib/url/tldList.ts @@ -1,4 +1,4 @@ -import TLDtxt from "./tlds.txt?raw"; +import TLDtxt from "./tlds.txt"; export function getTLD() { return TLDtxt.split("\r\n").filter((line) => line[0] !== "#"); diff --git a/lib/url/tlds.txt b/lib/url/tlds.txt index 143dcdc..d28a03d 100644 --- a/lib/url/tlds.txt +++ b/lib/url/tlds.txt @@ -1,4 +1,4 @@ -# Version 2024071300, Last Updated Sat Jul 13 07:07:01 2024 UTC +# Version 2025011900, Last Updated Sun Jan 19 07:07:01 2025 UTC AAA AARP ABB @@ -297,7 +297,6 @@ CY CYMRU CYOU CZ -DABUR DAD DANCE DATA @@ -1444,4 +1443,4 @@ ZIP ZM ZONE ZUERICH -ZW +ZW \ No newline at end of file diff --git a/lib/url/validLink.ts b/lib/url/validLink.ts index 851d676..b1e1afa 100644 --- a/lib/url/validLink.ts +++ b/lib/url/validLink.ts @@ -1,41 +1,33 @@ import { toASCII } from "tr46"; import { getTLD } from "./tldList"; +console.log(getTLD()); + export default function validLink(link: string) { let finalURL; try { - const url = new URL(link); - finalURL = url; + new URL(link); return true; } catch (error) { // if the URL is invalid, try to add the protocol try { - const urlWithHTTP = new URL("http://" + link); - finalURL = urlWithHTTP; + finalURL = new URL("http://" + link); } catch (error) { return false; } } if (finalURL.host.endsWith(".")) return false; - if ( - validTLD(finalURL.host) || + return validTLD(finalURL.host) || isValidIPv6(link.slice(1, finalURL.host.length - 1)) || - isValidIPv4(link) - ) { - return true; - } - return false; + isValidIPv4(link); + } export function validTLD(domain: string): boolean { if (!domain.includes(".")) return false; const tld = toASCII(domain.split(".").reverse()[0]); const tldList = getTLD(); - if (tldList.includes(tld.toUpperCase())) { - return true; - } else { - return false; - } + return !!tldList.includes(tld.toUpperCase()); } export function isValidIPv6(ip: string): boolean { @@ -69,10 +61,8 @@ export function isValidIPv6(ip: string): boolean { return false; } } - if (doubleColonCount === 0 && groups !== 8) { - return false; - } - return true; + return !(doubleColonCount === 0 && groups !== 8); + } export function isValidIPv4(ip: string): boolean { diff --git a/lib/version.ts b/lib/version.ts index 663e101..915fc20 100644 --- a/lib/version.ts +++ b/lib/version.ts @@ -1,8 +1,5 @@ import * as pjson from "package.json"; -export default function getVersion() { - return pjson.version; -} +export const clientVersion = pjson.version; -export const clientNLUVersion = 4; export const apiVersion = 1; diff --git a/package.json b/package.json index 194d1e8..d5fe039 100644 --- a/package.json +++ b/package.json @@ -1,65 +1,67 @@ { - "name": "sparkast", - "private": false, - "version": "5.8.1", - "type": "module", - "scripts": { - "dev": "bun server.ts", - "build": "bun license-gen && tsc -b && vite build", - "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", - "preview": "NODE_ENV=production bun server.ts", - "license-gen": "bunx generate-license-file --input package.json --output lib/license.txt --overwrite", - "format": "prettier --write ." - }, - "dependencies": { - "@alikia/search-complete": "^0.4.4", - "@iconify/react": "^5.0.1", - "@nextui-org/react": "^2.4.2", - "@types/bun": "^1.1.6", - "@types/express": "^4.17.21", - "@types/tr46": "^5.0.0", - "@xenova/transformers": "^2.17.2", - "cac": "^6.7.14", - "chalk": "^5.3.0", - "express": "^4.19.2", - "fflate": "^0.8.2", - "framer-motion": "^11.2.12", - "generate-license-file": "^3.5.1", - "i18next": "^23.11.5", - "i18next-browser-languagedetector": "^8.0.0", - "i18next-icu": "^2.3.0", - "jest": "^29.7.0", - "jotai": "^2.8.3", - "node-nlp": "^4.27.0", - "onnxruntime-web": "^1.20.0-dev.20240925-a47254eaef", - "react": "^18.3.1", - "react-dom": "^18.3.1", - "react-i18next": "^14.1.2", - "react-router": "^6.23.1", - "react-router-dom": "^6.23.1", - "tr46": "^5.0.0", - "valid-url": "^1.0.9", - "validate-color": "^2.2.4", - "vite-express": "^0.17.0" - }, - "devDependencies": { - "@types/react": "^18.3.3", - "@types/react-dom": "^18.3.0", - "@types/valid-url": "^1.0.7", - "@typescript-eslint/eslint-plugin": "^7.13.1", - "@typescript-eslint/parser": "^7.13.1", - "@vitejs/plugin-react-swc": "^3.5.0", - "autoprefixer": "^10.4.19", - "eslint": "^8.57.0", - "eslint-plugin-react-hooks": "^4.6.2", - "eslint-plugin-react-refresh": "^0.4.7", - "postcss": "^8.4.38", - "prettier": "^3.3.3", - "tailwindcss": "^3.4.4", - "typescript": "^5.2.2", - "vite": "^5.3.1", - "vite-plugin-chunk-split": "^0.5.0", - "vite-plugin-pages": "^0.32.2", - "vite-tsconfig-paths": "^4.3.2" - } + "name": "sparkast", + "private": false, + "version": "5.8.1", + "type": "module", + "scripts": { + "dev": "bun server.ts", + "build": "bun license-gen && tsc -b && vite build", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "preview": "NODE_ENV=production bun server.ts", + "license-gen": "bunx generate-license-file --input package.json --output lib/license.txt --overwrite", + "format": "prettier --write ." + }, + "dependencies": { + "@alikia/search-complete": "^0.4.4", + "@iconify/react": "^5.0.1", + "@nextui-org/react": "^2.4.2", + "@types/bun": "^1.1.6", + "@types/express": "^4.17.21", + "@types/tr46": "^5.0.0", + "@xenova/transformers": "^2.17.2", + "cac": "^6.7.14", + "chalk": "^5.3.0", + "express": "^4.19.2", + "fflate": "^0.8.2", + "framer-motion": "^11.2.12", + "generate-license-file": "^3.5.1", + "i18next": "^23.11.5", + "i18next-browser-languagedetector": "^8.0.0", + "i18next-icu": "^2.3.0", + "jest": "^29.7.0", + "jotai": "^2.8.3", + "node-nlp": "^4.27.0", + "onnxruntime-web": "^1.20.0-dev.20240925-a47254eaef", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-helmet": "^6.1.0", + "react-i18next": "^14.1.2", + "react-router": "^6.23.1", + "react-router-dom": "^6.23.1", + "tr46": "^5.0.0", + "valid-url": "^1.0.9", + "validate-color": "^2.2.4", + "vite-express": "^0.17.0" + }, + "devDependencies": { + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@types/react-helmet": "^6.1.11", + "@types/valid-url": "^1.0.7", + "@typescript-eslint/eslint-plugin": "^7.13.1", + "@typescript-eslint/parser": "^7.13.1", + "@vitejs/plugin-react-swc": "^3.5.0", + "autoprefixer": "^10.4.19", + "eslint": "^8.57.0", + "eslint-plugin-react-hooks": "^4.6.2", + "eslint-plugin-react-refresh": "^0.4.7", + "postcss": "^8.4.38", + "prettier": "^3.3.3", + "tailwindcss": "^3.4.4", + "typescript": "^5.2.2", + "vite": "^5.3.1", + "vite-plugin-chunk-split": "^0.5.0", + "vite-plugin-pages": "^0.32.2", + "vite-tsconfig-paths": "^4.3.2" + } } diff --git a/pages/about/index.tsx b/pages/about/index.tsx index 1ba9687..974522e 100644 --- a/pages/about/index.tsx +++ b/pages/about/index.tsx @@ -1,8 +1,38 @@ import useDarkMode from "lib/darkModeHook"; -import getVersion, { apiVersion, clientNLUVersion } from "lib/version"; +import { apiVersion, clientVersion } from "lib/version"; import AboutLayout from "./layout"; import { useTranslation } from "react-i18next"; +function License() { + const { t } = useTranslation(); + return ( + <> + <p className="flex items-center my-4"> + <span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]"> + {t("about.oss.title")} + </span> + <span + className="relative font-bold px-2 py-1 rounded-md text-nowrap underline + bg-green-600 text-white" + > + <a href="/about/oss-licenses">{t("about.oss.view")}</a> + </span> + </p> + <p className="flex items-center my-4"> + <span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]"> + {t("about.license.title")} + </span> + <span + className="relative font-bold px-2 py-1 rounded-md text-nowrap underline + bg-red-500 text-white" + > + <a href="/about/license">{t("about.license.text")}</a> + </span> + </p> + </> + ); +} + export default function AboutPage() { const darkMode = useDarkMode(); const { t } = useTranslation(); @@ -26,47 +56,41 @@ export default function AboutPage() { </div> </div> - <Version title="Frontend Version" version={getVersion()} versionClass="bg-red-500" /> <Version - title="Browser NLU Model Version" - version={"Build " + clientNLUVersion} + title={t("about.backend-api-version")} + version={"/api/v" + apiVersion} versionClass="bg-purple-500" /> + <Version - title="Backend API Version" - version={"/api/v" + apiVersion} + title={t("about.client-version")} + version={clientVersion} versionClass="bg-orange-500" /> - <p className="flex items-center my-3"> - <span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">License</span> - <span - className="relative px-2 py-1 text-sm font-bold rounded-md text-nowrap underline - bg-green-600 text-white" - > - <a href="/about/license">{t("about.license.view")}</a> - </span> - </p> - <p className="relative font-bold text-2xl mt-12">Presented By</p> + <License /> + + <p className="relative font-bold text-2xl mt-12">{t("about.presented-by")}</p> {!darkMode && ( <img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" /> )} {darkMode && ( - <img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-56 mt-6" /> + <img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-64 mt-6" /> )} </AboutLayout> ); } function Version(props: { title: string; version: string; versionClass?: string }) { - document.title = "About SparkHome"; const { t } = useTranslation(); return ( - <p className="flex items-center my-3"> - <span className="font-bold text-xl md:text-2xl mr-4 w-[36rem]">{t(props.title)}</span> + <p className="flex items-center my-4"> + <span className="font-medium text-2xl md:text-2xl mr-4 w-[36rem]"> + {t(props.title)} + </span> <span className={ - "relative px-2 py-1 text-sm font-bold rounded-md text-nowrap text-white " + + "relative px-2 py-1 font-bold rounded-md text-nowrap text-white " + props.versionClass || "" } > diff --git a/pages/about/layout.tsx b/pages/about/layout.tsx index 1a697bb..09aa4c9 100644 --- a/pages/about/layout.tsx +++ b/pages/about/layout.tsx @@ -1,6 +1,14 @@ +import React from "react"; +import { useTranslation } from "react-i18next"; +import { Helmet } from "react-helmet"; + export default function AboutLayout({ children }: { children: React.ReactNode }) { + const { t } = useTranslation(); return ( <div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]"> + <Helmet> + <title>{t('about.title')}</title> + </Helmet> <main className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4 pt-12 px-3 md:px-0" diff --git a/pages/about/license/index.tsx b/pages/about/license/OSS.tsx similarity index 51% rename from pages/about/license/index.tsx rename to pages/about/license/OSS.tsx index 7385ebe..b891635 100644 --- a/pages/about/license/index.tsx +++ b/pages/about/license/OSS.tsx @@ -1,13 +1,20 @@ import LICENSE from "lib/license.txt?raw"; +import { Helmet } from "react-helmet"; +import { useTranslation } from "react-i18next"; -export default function LicensePage() { +export default function OSSLicensesPage() { + const { t } = useTranslation(); return ( <div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden"> + <Helmet> + <title>{t("oss_license.title")}</title> + </Helmet> <main className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4 pt-12" > - <h1 className="text-4xl font-bold mb-6">LICENSE</h1> + <h1 className="text-4xl font-bold mb-6">{t('oss_license.page-title')}</h1> + <p className="text-lg mb-8">{t('oss_license.desc')}</p> <div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div> </main> </div> diff --git a/pages/about/license/ThisProject.tsx b/pages/about/license/ThisProject.tsx new file mode 100644 index 0000000..fc45951 --- /dev/null +++ b/pages/about/license/ThisProject.tsx @@ -0,0 +1,22 @@ +import LICENSE from "LICENSE?raw"; +import { useTranslation } from "react-i18next"; +import { Helmet } from "react-helmet"; + +export default function ThisProjectLicensePage() { + const { t } = useTranslation(); + return ( + <div className="dark:bg-[rgb(23,25,29)] dark:text-white min-h-screen w-screen overflow-x-hidden"> + <Helmet> + <title>{t("license.title")}</title> + </Helmet> + <main + className="relative h-full w-full md:w-3/4 lg:w-1/2 left-0 md:left-[12.5%] lg:left-1/4 + pt-12" + > + <h1 className="text-4xl font-bold mb-6">{t('license.page-title')}</h1> + <p className="text-lg mb-8">{t('license.desc')}</p> + <div className="font-mono text-justify whitespace-break-spaces">{LICENSE}</div> + </main> + </div> + ); +} diff --git a/pages/index.tsx b/pages/index.tsx index a7e8bfe..3303ff0 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -13,6 +13,7 @@ export default function Homepage() { return ( <div className="h-screen w-screen overflow-x-hidden bg-white dark:bg-[rgb(23,25,29)]"> + <title>sparkast</title> <Background /> <EngineSelector diff --git a/src/app.tsx b/src/app.tsx index 9bb0659..9c11ef3 100644 --- a/src/app.tsx +++ b/src/app.tsx @@ -3,7 +3,8 @@ import { createBrowserRouter, RouterProvider } from "react-router-dom"; import "./i18n"; import Homepage from "pages"; import AboutPage from "pages/about"; -import LicensePage from "pages/about/license"; +import OSSLicensesPage from "pages/about/license/OSS"; +import ThisProjectLicensePage from "../pages/about/license/ThisProject.tsx"; const router = createBrowserRouter([ { @@ -11,14 +12,16 @@ const router = createBrowserRouter([ element: <Homepage /> }, { - path: "about", + path: "/about", element: <AboutPage />, - children: [ - { - path: "license", - element: <LicensePage /> - } - ] + }, + { + path: "/about/oss-licenses", + element: <OSSLicensesPage /> + }, + { + path: "/about/license", + element: <ThisProjectLicensePage /> } ]); diff --git a/test/bytesToUnicode.test.ts b/test/bytesToUnicode.test.ts deleted file mode 100644 index 18bfa8f..0000000 --- a/test/bytesToUnicode.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { expect, test } from "bun:test"; -import bytesToUnicodes from "../lib/nlp/tokenize/bytesToUnicode"; - -test("bytesToUnicode: test", () => { - expect(bytesToUnicodes("Hello 你好")).toEqual("HelloĠä½łå¥½"); -}); diff --git a/test/getEmbeddings.test.ts b/test/getEmbeddings.test.ts deleted file mode 100644 index e69de29..0000000 diff --git a/test/unicodeToBytes.test.ts b/test/unicodeToBytes.test.ts deleted file mode 100644 index ab5be6f..0000000 --- a/test/unicodeToBytes.test.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { expect, test } from "bun:test"; -import unicodeToBytes from "../lib/nlp/tokenize/unicodeToBytes"; - -test("unicodeToBytes: test", () => { - expect(unicodeToBytes("HelloĠä½łå¥½")).toEqual("Hello 你好"); -}); diff --git a/test/validLink.test.ts b/test/validLink.test.ts index 58e1aa1..a028f6b 100644 --- a/test/validLink.test.ts +++ b/test/validLink.test.ts @@ -60,6 +60,7 @@ describe("Check if the given TLD exist and assigned.", () => { expect(validTLD("example.foo")).toBe(true); expect(validTLD("example.bar")).toBe(true); expect(validTLD("example.zip")).toBe(true); + expect(validTLD("xn--s8w913fdga.chn.moe")).toBe(true); }); test("Exist but not assigned TLD", () => { expect(validTLD("example.active")).toBe(false);