diff options
| -rw-r--r-- | js/fsa-tokenizer.js | 11 |
1 files changed, 5 insertions, 6 deletions
diff --git a/js/fsa-tokenizer.js b/js/fsa-tokenizer.js index 601583c..6ff4ef7 100644 --- a/js/fsa-tokenizer.js +++ b/js/fsa-tokenizer.js @@ -7,7 +7,7 @@ const TokenizerFSA = (() => { IN_SYMBOL: 'IN_SYMBOL' }; - // Utility functions to check character types + // Check character types const isLetter = char => /[a-zA-Z]/.test(char); const isDigit = char => /\d/.test(char); const isSymbol = char => /\W/.test(char) && !/\s/.test(char); @@ -17,7 +17,7 @@ const TokenizerFSA = (() => { if (token) tokens.push(token); }; - // Process a single character and update the state and token accordingly + // Process a single character and update the state and token const processChar = (state, char, token, tokens) => { switch (state) { case states.START: @@ -58,17 +58,16 @@ const TokenizerFSA = (() => { token = result.token; } - // Add the last token if any + // Add the last token if one exists addToken(token, tokens); return tokens; }; - // Expose the tokenize function as a public method return { tokenize }; })(); -// Example usage +// example usage const text = "Hello, world! 123"; const tokens = TokenizerFSA.tokenize(text); -console.log(tokens); // Output: ['Hello', ',', 'world', '!', '123'] +console.log(tokens); // Ought to output: ['Hello', ',', 'world', '!', '123'] |