From f7de462a28d49f59851efb76306bde45dc6aea8b Mon Sep 17 00:00:00 2001
From: alikia2x <alikia2x@outlook.com>
Date: Mon, 20 Jan 2025 03:03:28 +0800
Subject: [PATCH] fix: unable to properly segment raw TLD data feat:
 searchbox's value changes while selecting suggestion

---
 components/onesearch/onesearch.tsx |  6 ++++--
 components/search.tsx              | 25 ++++++++++++++++++-------
 lib/state/searchboxLastInput.ts    |  5 +++++
 lib/url/tldList.ts                 |  2 +-
 test/tokenize.test.ts              | 13 -------------
 5 files changed, 28 insertions(+), 23 deletions(-)
 create mode 100644 lib/state/searchboxLastInput.ts
 delete mode 100644 test/tokenize.test.ts

diff --git a/components/onesearch/onesearch.tsx b/components/onesearch/onesearch.tsx
index 66e8b9a..6f85277 100644
--- a/components/onesearch/onesearch.tsx
+++ b/components/onesearch/onesearch.tsx
@@ -23,6 +23,7 @@ import { loadVocab } from "lib/nlp/tokenize/loadVocab";
 import BPETokenizer from "lib/nlp/tokenize/BPEtokenizer";
 import energyScore from "lib/nlp/energyScore";
 import bytesToUnicode from "lib/nlp/tokenize/bytesToUnicode";
+import { searchboxLastInputAtom } from "lib/state/searchboxLastInput";
 
 interface EmbeddingLayer {
 	[key: number]: Float32Array<ArrayBufferLike>;
@@ -33,6 +34,7 @@ export default function OneSearch() {
 	const [embeddingLayer, setEmbeddingLayer] = useState<EmbeddingLayer | null>(null);
 	const [NLUsession, setNLUsession] = useState<ort.InferenceSession | null>(null);
 	const [tokenizer, setTokenizer] = useState<BPETokenizer | null>(null);
+	const lastInput = useAtomValue(searchboxLastInputAtom);
 	const lastRequestTimeRef = useRef(0);
 	const selected = useAtomValue(selectedSuggestionAtom);
 	const settings = useAtomValue(settingsAtom);
@@ -69,7 +71,7 @@ export default function OneSearch() {
 				// Handle fetch error
 				sendError(error);
 			});
-	}, [query]);
+	}, [lastInput]);
 
 	function updateSuggestion(data: suggestionItem[]) {
 		setFinalSuggetsion((cur: suggestionItem[]) => {
@@ -177,7 +179,7 @@ export default function OneSearch() {
 			}
 			console.log(data, energyScore(data));
 		})();
-	}, [query, engineName]);
+	}, [lastInput, engineName]);
 
 	return (
 		<SuggestionBox>
diff --git a/components/search.tsx b/components/search.tsx
index 0bae01b..5dedf83 100644
--- a/components/search.tsx
+++ b/components/search.tsx
@@ -6,12 +6,14 @@ import { selectedSuggestionAtom } from "lib/state/suggestionSelection";
 import handleEnter from "lib/onesearch/handleEnter";
 import { suggestionAtom } from "lib/state/suggestion";
 import { useTranslation } from "react-i18next";
+import { searchboxLastInputAtom } from "lib/state/searchboxLastInput";
 
 export default function Search(props: { onFocus: () => void }) {
 	const { t } = useTranslation();
 	const settings = useAtomValue(settingsAtom);
 	const [query, setQuery] = useAtom(queryAtom);
 	const [selectedSuggestion, setSelected] = useAtom(selectedSuggestionAtom);
+	const [_, setLastInput] = useAtom(searchboxLastInputAtom)
 	const suggestions = useAtomValue(suggestionAtom);
 	const searchBoxRef = useRef<HTMLInputElement>(null);
 
@@ -25,11 +27,21 @@ export default function Search(props: { onFocus: () => void }) {
 		} else if (e.key == "ArrowUp") {
 			e.preventDefault();
 			const len = suggestions.length;
-			setSelected((selectedSuggestion - 1 + len) % len);
+			const lastSelectedIndex = (selectedSuggestion - 1 + len) % len;
+			const lastSuggeston = suggestions[lastSelectedIndex];
+			setSelected(lastSelectedIndex);
+			if (["QUERY", "NAVIGATION", "default"].includes(lastSuggeston.type)) {
+				setQuery(lastSuggeston.suggestion);
+			}
 		} else if (e.key == "ArrowDown") {
 			e.preventDefault();
 			const len = suggestions.length;
-			setSelected((selectedSuggestion + 1) % len);
+			const nextSelectedIndex = (selectedSuggestion + 1 + len) % len;
+			const nextSuggeston = suggestions[nextSelectedIndex];
+			setSelected(nextSelectedIndex);
+			if (["QUERY", "NAVIGATION", "default"].includes(nextSuggeston.type)) {
+				setQuery(nextSuggeston.suggestion);
+			}
 		}
 	}
 
@@ -48,11 +60,10 @@ export default function Search(props: { onFocus: () => void }) {
 					placeholder={t("search.placeholder")}
 					onFocus={props.onFocus}
 					onKeyDown={handleKeydown}
-					onChange={(e) =>
-						setQuery(() => {
-							return e.target.value;
-						})
-					}
+					onChange={(e) => {
+						setLastInput(new Date().getTime());
+						setQuery(() => e.target.value);
+					}}
 					autoComplete="off"
 					autoCorrect="off"
 					autoCapitalize="off"
diff --git a/lib/state/searchboxLastInput.ts b/lib/state/searchboxLastInput.ts
new file mode 100644
index 0000000..619497f
--- /dev/null
+++ b/lib/state/searchboxLastInput.ts
@@ -0,0 +1,5 @@
+import { atom } from "jotai";
+
+const searchboxLastInputAtom = atom(0);
+
+export { searchboxLastInputAtom };
diff --git a/lib/url/tldList.ts b/lib/url/tldList.ts
index 0868bca..950ae99 100644
--- a/lib/url/tldList.ts
+++ b/lib/url/tldList.ts
@@ -1,5 +1,5 @@
 import TLDtxt from "./tlds.txt?raw";
 
 export function getTLD() {
-	return TLDtxt.split("\n").filter((line) => line[0] !== "#");
+	return TLDtxt.split("\r\n").filter((line) => line[0] !== "#");
 }
diff --git a/test/tokenize.test.ts b/test/tokenize.test.ts
deleted file mode 100644
index cdf6476..0000000
--- a/test/tokenize.test.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import tokenize from "../lib/nlp/tokenize/tokenizer";
-
-describe("Test if tokenizer works", () => {
-	test("Using without a mirror", async () => {
-		let result = await tokenize("你好，世界！", "Qwen/Qwen2.5-3B", false);
-		expect(result).toEqual([108386, 3837, 99489, 6313]);
-	});
-	test("Using with a mirror", async () => {
-		let result = await tokenize("你好，世界！", "Qwen/Qwen2.5-3B", true);
-		expect(result).toEqual([108386, 3837, 99489, 6313]);
-	});
-});