ref: project name, start script
This commit is contained in:
parent
bf92c349fc
commit
43da50f37c
1
.prettierignore
Normal file
1
.prettierignore
Normal file
@ -0,0 +1 @@
|
|||||||
|
public/*
|
@ -110,13 +110,12 @@ export default function OneSearch() {
|
|||||||
})();
|
})();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (tokenizer !== null) return;
|
if (tokenizer !== null) return;
|
||||||
(async function () {
|
(async function () {
|
||||||
await loadTokenizer();
|
await loadTokenizer();
|
||||||
})();
|
})();
|
||||||
},[]);
|
}, []);
|
||||||
|
|
||||||
async function loadModel(modelPath: string) {
|
async function loadModel(modelPath: string) {
|
||||||
ort.env.wasm.wasmPaths = "/onnx/";
|
ort.env.wasm.wasmPaths = "/onnx/";
|
||||||
|
@ -1,81 +1,81 @@
|
|||||||
class TrieNode {
|
class TrieNode {
|
||||||
children: Map<string, TrieNode>;
|
children: Map<string, TrieNode>;
|
||||||
tokenId: number | null;
|
tokenId: number | null;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.children = new Map();
|
this.children = new Map();
|
||||||
this.tokenId = null;
|
this.tokenId = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class Trie {
|
class Trie {
|
||||||
root: TrieNode;
|
root: TrieNode;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.root = new TrieNode();
|
this.root = new TrieNode();
|
||||||
}
|
}
|
||||||
|
|
||||||
insert(token: string, tokenId: number) {
|
insert(token: string, tokenId: number) {
|
||||||
let node = this.root;
|
let node = this.root;
|
||||||
for (const char of token) {
|
for (const char of token) {
|
||||||
if (!node.children.has(char)) {
|
if (!node.children.has(char)) {
|
||||||
node.children.set(char, new TrieNode());
|
node.children.set(char, new TrieNode());
|
||||||
}
|
}
|
||||||
node = node.children.get(char)!;
|
node = node.children.get(char)!;
|
||||||
}
|
}
|
||||||
node.tokenId = tokenId;
|
node.tokenId = tokenId;
|
||||||
}
|
}
|
||||||
|
|
||||||
searchLongestToken(text: string): [number | null, number] {
|
searchLongestToken(text: string): [number | null, number] {
|
||||||
let node = this.root;
|
let node = this.root;
|
||||||
let longestTokenId: number | null = null;
|
let longestTokenId: number | null = null;
|
||||||
let currentTokenLength = 0;
|
let currentTokenLength = 0;
|
||||||
|
|
||||||
for (const char of text) {
|
for (const char of text) {
|
||||||
if (!node.children.has(char)) {
|
if (!node.children.has(char)) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
node = node.children.get(char)!;
|
node = node.children.get(char)!;
|
||||||
currentTokenLength += 1;
|
currentTokenLength += 1;
|
||||||
if (node.tokenId !== null) {
|
if (node.tokenId !== null) {
|
||||||
longestTokenId = node.tokenId;
|
longestTokenId = node.tokenId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return [longestTokenId, currentTokenLength];
|
return [longestTokenId, currentTokenLength];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export default class BPETokenizer {
|
export default class BPETokenizer {
|
||||||
private trie: Trie;
|
private trie: Trie;
|
||||||
|
|
||||||
constructor(vocabulary: { [key: string]: number }) {
|
constructor(vocabulary: { [key: string]: number }) {
|
||||||
this.trie = new Trie();
|
this.trie = new Trie();
|
||||||
for (const token in vocabulary) {
|
for (const token in vocabulary) {
|
||||||
if (vocabulary.hasOwnProperty(token)) {
|
if (vocabulary.hasOwnProperty(token)) {
|
||||||
this.trie.insert(token, vocabulary[token]);
|
this.trie.insert(token, vocabulary[token]);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tokenize(text: string): number[] {
|
tokenize(text: string): number[] {
|
||||||
const tokenIds: number[] = [];
|
const tokenIds: number[] = [];
|
||||||
let i = 0;
|
let i = 0;
|
||||||
|
|
||||||
while (i < text.length) {
|
while (i < text.length) {
|
||||||
const [longestTokenId, length] = this.trie.searchLongestToken(text.slice(i));
|
const [longestTokenId, length] = this.trie.searchLongestToken(text.slice(i));
|
||||||
if (longestTokenId !== null) {
|
if (longestTokenId !== null) {
|
||||||
tokenIds.push(longestTokenId);
|
tokenIds.push(longestTokenId);
|
||||||
i += length;
|
i += length;
|
||||||
} else {
|
} else {
|
||||||
// If no token is found, treat the character as a single token
|
// If no token is found, treat the character as a single token
|
||||||
tokenIds.push(text.charCodeAt(i));
|
tokenIds.push(text.charCodeAt(i));
|
||||||
i += 1;
|
i += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return tokenIds;
|
return tokenIds;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Example usage:
|
// Example usage:
|
||||||
@ -91,4 +91,4 @@ export default class BPETokenizer {
|
|||||||
|
|
||||||
// const text = 'ababbaa';
|
// const text = 'ababbaa';
|
||||||
// const tokenIds = tokenizer.tokenize(text);
|
// const tokenIds = tokenizer.tokenize(text);
|
||||||
// console.log(tokenIds); // Output: [ 3, 3, 6, 1 ]
|
// console.log(tokenIds); // Output: [ 3, 3, 6, 1 ]
|
||||||
|
22
lib/server/startScript.ts
Normal file
22
lib/server/startScript.ts
Normal file
@ -0,0 +1,22 @@
|
|||||||
|
import { promises as dns } from "node:dns";
|
||||||
|
|
||||||
|
// Copied from vite/src/node/utils.ts
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns resolved localhost address when `dns.lookup` result differs from DNS
|
||||||
|
*
|
||||||
|
* `dns.lookup` result is same when defaultResultOrder is `verbatim`.
|
||||||
|
* Even if defaultResultOrder is `ipv4first`, `dns.lookup` result maybe same.
|
||||||
|
* For example, when IPv6 is not supported on that machine/network.
|
||||||
|
*/
|
||||||
|
export async function getLocalhostAddressIfDiffersFromDNS(): Promise<string | undefined> {
|
||||||
|
const [nodeResult, dnsResult] = await Promise.all([
|
||||||
|
dns.lookup("localhost"),
|
||||||
|
dns.lookup("localhost", { verbatim: true })
|
||||||
|
]);
|
||||||
|
const isSame =
|
||||||
|
nodeResult.family === dnsResult.family && nodeResult.address === dnsResult.address;
|
||||||
|
return isSame ? undefined : nodeResult.address;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const wildcardHosts = new Set(["0.0.0.0", "::", "0000:0000:0000:0000:0000:0000:0000:0000"]);
|
@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"name": "sparkhome",
|
"name": "sparkast",
|
||||||
"private": false,
|
"private": false,
|
||||||
"version": "5.8.1",
|
"version": "5.8.1",
|
||||||
"type": "module",
|
"type": "module",
|
||||||
|
@ -48,8 +48,12 @@ export default function AboutPage() {
|
|||||||
</p>
|
</p>
|
||||||
|
|
||||||
<p className="relative font-bold text-2xl mt-12">Presented By</p>
|
<p className="relative font-bold text-2xl mt-12">Presented By</p>
|
||||||
{!darkMode && <img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" />}
|
{!darkMode && (
|
||||||
{darkMode && <img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-56 mt-6" />}
|
<img src="/assets/img/LuminaraStudio.png" className="relative md:h-64 mt-6" />
|
||||||
|
)}
|
||||||
|
{darkMode && (
|
||||||
|
<img src="/assets/img/LuminaraStudioDark.png" className="relative md:h-56 mt-6" />
|
||||||
|
)}
|
||||||
</AboutLayout>
|
</AboutLayout>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
119
server.ts
119
server.ts
@ -5,60 +5,32 @@ import pjson from "./package.json";
|
|||||||
import { networkInterfaces } from "os";
|
import { networkInterfaces } from "os";
|
||||||
import cac from "cac";
|
import cac from "cac";
|
||||||
import { configureBackendRoutes } from "./backend/route";
|
import { configureBackendRoutes } from "./backend/route";
|
||||||
|
import { Server, IncomingMessage, ServerResponse } from "http";
|
||||||
|
import { getLocalhostAddressIfDiffersFromDNS, wildcardHosts } from "lib/server/startScript";
|
||||||
|
|
||||||
async function helloMessage() {
|
async function helloMessage() {
|
||||||
const { base } = await ViteExpress.getViteConfig();
|
const { base } = await ViteExpress.getViteConfig();
|
||||||
const timeCost = new Date().getTime() - start.getTime();
|
const timeCost = Date.now() - start.getTime();
|
||||||
console.log("");
|
|
||||||
console.log(
|
console.log(
|
||||||
" ",
|
`\n ${chalk.redBright("sparkast v" + pjson.version)} ${chalk.whiteBright("ready in")} ${Math.round(timeCost)} ms\n`
|
||||||
chalk.redBright("SparkHome"),
|
|
||||||
chalk.redBright("v" + pjson.version),
|
|
||||||
chalk.whiteBright(" ready in"),
|
|
||||||
`${Math.round(timeCost)} ms`
|
|
||||||
);
|
);
|
||||||
console.log("");
|
console.log(` ${chalk.redBright("➜ Local:")} ${chalk.cyan(`http://${name}:${port}${base}`)}`);
|
||||||
console.log(
|
if (host === undefined) {
|
||||||
" ",
|
ips.forEach((ip) =>
|
||||||
chalk.redBright("➜ "),
|
|
||||||
"Local:\t",
|
|
||||||
chalk.cyan(`http://${host}:${port}${base}`)
|
|
||||||
);
|
|
||||||
if (host !== "localhost") {
|
|
||||||
for (const ip of ips) {
|
|
||||||
console.log(
|
console.log(
|
||||||
" ",
|
` ${chalk.redBright("➜ Network:")} ${chalk.cyan(`http://${ip}:${port}${base}`)}`
|
||||||
chalk.redBright("➜ "),
|
)
|
||||||
"Network:\t",
|
);
|
||||||
chalk.cyan(`http://${ip}:${port}${base}`)
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
console.log(
|
console.log(` ${chalk.red("➜ ")}${chalk.whiteBright("press h + enter to show help")}`);
|
||||||
" ",
|
|
||||||
chalk.red("➜ "),
|
|
||||||
chalk.whiteBright("press"),
|
|
||||||
"h + enter",
|
|
||||||
chalk.whiteBright("to show help")
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function handleInput() {
|
async function handleInput(server: Server<typeof IncomingMessage, typeof ServerResponse>) {
|
||||||
for await (const line of console) {
|
for await (const line of console) {
|
||||||
switch (line) {
|
switch (line.trim()) {
|
||||||
case "h":
|
case "h":
|
||||||
console.log(" Shortcuts");
|
|
||||||
console.log(
|
console.log(
|
||||||
" ",
|
` Shortcuts\n ${chalk.whiteBright("press c + enter to clear console")}\n ${chalk.whiteBright("press q + enter to quit")}`
|
||||||
chalk.whiteBright("press"),
|
|
||||||
"c + enter ",
|
|
||||||
chalk.whiteBright("to clear console")
|
|
||||||
);
|
|
||||||
console.log(
|
|
||||||
" ",
|
|
||||||
chalk.whiteBright("press"),
|
|
||||||
"q + enter ",
|
|
||||||
chalk.whiteBright("to quit")
|
|
||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
case "c":
|
case "c":
|
||||||
@ -68,8 +40,6 @@ async function handleInput() {
|
|||||||
server.on("vite:close", () => {});
|
server.on("vite:close", () => {});
|
||||||
server.close();
|
server.close();
|
||||||
return;
|
return;
|
||||||
default:
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -77,41 +47,48 @@ async function handleInput() {
|
|||||||
const start = new Date();
|
const start = new Date();
|
||||||
const cli = cac();
|
const cli = cac();
|
||||||
const nets = networkInterfaces();
|
const nets = networkInterfaces();
|
||||||
const ips: string[] = [];
|
const ips: string[] = Object.values(nets)
|
||||||
for (const name of Object.keys(nets)) {
|
.flat()
|
||||||
if (nets[name] === undefined) {
|
.filter(
|
||||||
continue;
|
(net) =>
|
||||||
}
|
!!net && net.family === (typeof net.family === "string" ? "IPv4" : 4) && !net.internal
|
||||||
for (const net of nets[name]) {
|
)
|
||||||
// Skip over non-IPv4 and internal (i.e. 127.0.0.1) addresses
|
.map((net) => (!!net ? net.address : undefined))
|
||||||
// 'IPv4' is in Node <= 17, from 18 it's a number 4 or 6
|
.filter((v) => v !== undefined);
|
||||||
const familyV4Value = typeof net.family === "string" ? "IPv4" : 4;
|
|
||||||
if (net.family === familyV4Value && !net.internal) {
|
|
||||||
ips.push(net.address);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const app = express();
|
const app = express();
|
||||||
const port = 3000;
|
const port = 3000;
|
||||||
let host = "localhost";
|
|
||||||
|
|
||||||
cli.option("--host [host]", "Sepcify host name");
|
let host: string | undefined = "localhost";
|
||||||
cli.help();
|
|
||||||
cli.version(pjson.version);
|
cli.option("--host [host]", "Specify host name").help().version(pjson.version);
|
||||||
const parsed = cli.parse();
|
const parsed = cli.parse();
|
||||||
if (
|
const optionsHost: string | boolean | undefined = parsed.options.host;
|
||||||
parsed.options.host !== undefined &&
|
|
||||||
typeof parsed.options.host == "boolean" &&
|
if (optionsHost === undefined || optionsHost === false) {
|
||||||
parsed.options.host
|
// Use a secure default
|
||||||
) {
|
host = "localhost";
|
||||||
host = "0.0.0.0";
|
} else if (optionsHost === true) {
|
||||||
|
// If passed --host in the CLI without arguments
|
||||||
|
host = undefined; // undefined typically means 0.0.0.0 or :: (listen on all IPs)
|
||||||
|
} else {
|
||||||
|
host = optionsHost;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set host name to localhost when possible
|
||||||
|
let name = host === undefined || wildcardHosts.has(host) ? "localhost" : host;
|
||||||
|
|
||||||
|
if (host === "localhost") {
|
||||||
|
const localhostAddr = await getLocalhostAddressIfDiffersFromDNS();
|
||||||
|
if (localhostAddr) {
|
||||||
|
name = localhostAddr;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
configureBackendRoutes(app);
|
configureBackendRoutes(app);
|
||||||
|
// if the var `host` is undefined, then just not pass it.
|
||||||
const server = app.listen(port, host);
|
const server = host ? app.listen(port, host) : app.listen(port);
|
||||||
|
|
||||||
ViteExpress.bind(app, server, helloMessage);
|
ViteExpress.bind(app, server, helloMessage);
|
||||||
|
|
||||||
handleInput();
|
handleInput(server);
|
||||||
|
Loading…
Reference in New Issue
Block a user