From e7f6f69dfe84082459c13f5e06a402051d4a77d0 Mon Sep 17 00:00:00 2001 From: Alikia2x Date: Mon, 17 Jun 2024 03:11:17 +0800 Subject: [PATCH] temp: NLP base class --- lib/nlp/base.ts | 19 +++++++++++++++++++ lib/nlp/stopwords.ts | 10 +--------- lib/onesearch/NLPResult.ts | 8 ++++---- 3 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 lib/nlp/base.ts diff --git a/lib/nlp/base.ts b/lib/nlp/base.ts new file mode 100644 index 0000000..c8abaf2 --- /dev/null +++ b/lib/nlp/base.ts @@ -0,0 +1,19 @@ +import { NLPResult } from "../onesearch/NLPResult"; +import { stopwords } from "./stopwords"; + +class NLP { + result: NLPResult; + constructor( + public query: String, + public task: String + ) { + this.result = new NLPResult(); + } + public removeStopwords(str: string, extraStopwords: string[] = [], disableDefault: boolean = false){ + const list = disableDefault ? extraStopwords : stopwords.concat(extraStopwords); + for (let word of list){ + str = str.replace(new RegExp(`\\b${word}\\b`, 'gi'), ''); + } + return str; + } +} \ No newline at end of file diff --git a/lib/nlp/stopwords.ts b/lib/nlp/stopwords.ts index c029189..0276f3a 100644 --- a/lib/nlp/stopwords.ts +++ b/lib/nlp/stopwords.ts @@ -1,9 +1 @@ -const stopwords = ["a","about","above","after","again","against","all","am","an","and","any","are","aren't","as","at","be","because","been","before","being","below","between","both","but","by","can't","cannot","could","couldn't","did","didn't","do","does","doesn't","doing","don't","down","during","each","few","for","from","further","had","hadn't","has","hasn't","have","haven't","having","he","he'd","he'll","he's","her","here","here's","hers","herself","him","himself","his","how","how's","i","i'd","i'll","i'm","i've","if","in","into","is","isn't","it","it's","its","itself","let's","me","more","most","mustn't","my","myself","no","nor","not","of","off","on","once","only","or","other","ought","our","ours ourselves","out","over","own","same","shan't","she","she'd","she'll","she's","should","shouldn't","so","some","such","than","that","that's","the","their","theirs","them","themselves","then","there","there's","these","they","they'd","they'll","they're","they've","this","those","through","to","too","under","until","up","very","was","wasn't","we","we'd","we'll","we're","we've","were","weren't","what","what's","when","when's","where","where's","which","while","who","who's","whom","why","why's","with","won't","would","wouldn't","you","you'd","you'll","you're","you've","your","yours","yourself","yourselves"]; - -export default function removeStopwords(str: string, extraStopwords: string[] = [], disableDefault: boolean = false){ - const list = disableDefault ? extraStopwords : stopwords.concat(extraStopwords); - for (let word of list){ - str = str.replace(new RegExp(`\\b${word}\\b`, 'gi'), ''); - } - return str; -} \ No newline at end of file +export const stopwords = ["a","about","above","after","again","against","all","am","an","and","any","are","aren't","as","at","be","because","been","before","being","below","between","both","but","by","can't","cannot","could","couldn't","did","didn't","do","does","doesn't","doing","don't","down","during","each","few","for","from","further","had","hadn't","has","hasn't","have","haven't","having","he","he'd","he'll","he's","her","here","here's","hers","herself","him","himself","his","how","how's","i","i'd","i'll","i'm","i've","if","in","into","is","isn't","it","it's","its","itself","let's","me","more","most","mustn't","my","myself","no","nor","not","of","off","on","once","only","or","other","ought","our","ours ourselves","out","over","own","same","shan't","she","she'd","she'll","she's","should","shouldn't","so","some","such","than","that","that's","the","their","theirs","them","themselves","then","there","there's","these","they","they'd","they'll","they're","they've","this","those","through","to","too","under","until","up","very","was","wasn't","we","we'd","we'll","we're","we've","were","weren't","what","what's","when","when's","where","where's","which","while","who","who's","whom","why","why's","with","won't","would","wouldn't","you","you'd","you'll","you're","you've","your","yours","yourself","yourselves"]; \ No newline at end of file diff --git a/lib/onesearch/NLPResult.ts b/lib/onesearch/NLPResult.ts index 6a67a0f..ab477c9 100644 --- a/lib/onesearch/NLPResult.ts +++ b/lib/onesearch/NLPResult.ts @@ -1,9 +1,9 @@ export class NLPResult { constructor( - public suggestion: string | null, - public intention: string | null, - public probability: number, - public confidence: number, + public suggestion: string | null = null, + public intention: string | null = null, + public probability: number = 0, + public confidence: number = 0, public relevanceBase: number = 2000, public confidenceWeight: number = 0.2, public type: string = "text",