8 lines
171 B
TypeScript
8 lines
171 B
TypeScript
import BPETokenizer from "./BPEtokenizer";
|
|
|
|
async function tokenize(text: string, tokenizer: BPETokenizer) {
|
|
return tokenizer.tokenize(text);
|
|
}
|
|
|
|
export default tokenize;
|