1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
|
const TokenizerFSA = (() => {
// Define the states
const states = {
START: 'START',
IN_WORD: 'IN_WORD',
IN_NUMBER: 'IN_NUMBER',
IN_SYMBOL: 'IN_SYMBOL'
};
// Check character types
const isLetter = char => /[a-zA-Z]/.test(char);
const isDigit = char => /\d/.test(char);
const isSymbol = char => /\W/.test(char) && !/\s/.test(char);
// Add a token to the list if it's not empty
const addToken = (token, tokens) => {
if (token) tokens.push(token);
};
// Process a single character and update the state and token
const processChar = (state, char, token, tokens) => {
switch (state) {
case states.START:
if (isLetter(char)) return { state: states.IN_WORD, token: char };
if (isDigit(char)) return { state: states.IN_NUMBER, token: char };
if (isSymbol(char)) return { state: states.IN_SYMBOL, token: char };
break;
case states.IN_WORD:
if (isLetter(char)) return { state, token: token + char };
addToken(token, tokens);
return { state: states.START, token: '' };
case states.IN_NUMBER:
if (isDigit(char)) return { state, token: token + char };
addToken(token, tokens);
return { state: states.START, token: '' };
case states.IN_SYMBOL:
if (isSymbol(char)) return { state, token: token + char };
addToken(token, tokens);
return { state: states.START, token: '' };
default:
addToken(token, tokens);
return { state: states.START, token: '' };
}
// Safety net - this shouldn't be reached unless something is wrong with one of the cases
return { state: states.START, token: '' }; // TODO: I've got a sneak suspicion this is being reached
};
// Tokenize the entire input text
const tokenize = text => {
let state = states.START;
let token = '';
const tokens = [];
for (const char of text) {
const result = processChar(state, char, token, tokens);
state = result.state;
token = result.token;
}
// Add the last token if one exists
addToken(token, tokens);
return tokens;
};
return { tokenize };
})();
// example usage
const text = "Oh my goodness! What an enormous banana, there must be 11 of them on that tr33!";
const tokens = TokenizerFSA.tokenize(text);
console.log(tokens);
// Output ought to be:
// [
// "Oh", "my", "goodness", "What", "an", "enormous", "banana", "there", "must", "be", "11", "of", "them", "on",
// "that", "tr", "33", "!"
// ]
|