summary refs log blame commit diff stats
path: root/tests/misc/trawstr.nim
blob: ab2aae1593755b39f99ff0666d8c0e576bfd9a64 (plain) (tree) 
a690e7b26 ^












1
2
3
4
5
6
7
8
9
10
11
12












                                 
discard """
  file: "trawstr.nim"
  line: 10
  errormsg: "closing \" expected"
"""
# Test the new raw strings:

const
  xxx = r"This is a raw string!"
  yyy = "This not\" #ERROR
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-02-11 10:07:29 +0000
n1'>1 2 3 4 5 6 7 8 9 
10

11
12
13
14
15
16
17
18
19

20

21
22
23
24
25
26

27

28
29
30
31
32
33
34
35
36
37
38

39

40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60

61

62
63
64
65
66

67
68
69

70
71

72

73
74
75
76
77

78

79
80
81









                             
                                               








                                                               
                                                                          





                                                                            
                                                                                                          










                                                                  
                                                                        




















                                                                  
                                




                            


                      

                                                                                                 
                                           




                                                                                   
 


                                             
const TokenizerFSA = (() => {
  // Define the states
  const states = {
    START: 'START',
    IN_WORD: 'IN_WORD',
    IN_NUMBER: 'IN_NUMBER',
    IN_SYMBOL: 'IN_SYMBOL'
  };

  // Utility functions to check character types
  const isLetter = char => /[a-zA-Z]/.test(char);
  const isDigit = char => /\d/.test(char);
  const isSymbol = char => /\W/.test(char) && !/\s/.test(char);

  // Add a token to the list if it's not empty
  const addToken = (token, tokens) => {
    if (token) tokens.push(token);
  };

  // Process a single character and update the state and token accordingly
  const processChar = (state, char, token, tokens) => {
    switch (state) {
      case states.START:
        if (isLetter(char)) return { state: states.IN_WORD, token: char };
        if (isDigit(char)) return { state: states.IN_NUMBER, token: char };
        if (isSymbol(char)) return { state: states.IN_SYMBOL, token: char };
        return { state: states.START, token: '' }; // Remain in START state for spaces or other characters

      case states.IN_WORD:
        if (isLetter(char)) return { state, token: token + char };
        addToken(token, tokens);
        return { state: states.START, token: '' };

      case states.IN_NUMBER:
        if (isDigit(char)) return { state, token: token + char };
        addToken(token, tokens);
        return { state: states.START, token: '' };

      case states.IN_SYMBOL: // FIXME: symbols aren't being well handled
        if (isSymbol(char)) return { state, token: token + char };
        addToken(token, tokens);
        return { state: states.START, token: '' };

      default:
        return { state: states.START, token: '' };
    }
  };

  // Tokenize the entire input text
  const tokenize = text => {
    let state = states.START;
    let token = '';
    const tokens = [];

    for (const char of text) {
      const result = processChar(state, char, token, tokens);
      state = result.state;
      token = result.token;
    }

    // Add the last token if any
    addToken(token, tokens);

    return tokens;
  };

  return { tokenize };
})();

// FIXME: symbols aren't being well handled, and consecutive, identical numbers are being ignored
const text = "Hello, world! 123";
const tokens = TokenizerFSA.tokenize(text);
console.log(tokens);  // Output: ['Hello', ',', 'world', '!', '123']

const text2 = "Oh no, it's 3.14159! Is it delicious? Is it mysterious? Who knows!";
const tokens2 = TokenizerFSA.tokenize(text2);
console.log(tokens2);

const text3 = "über l337 hard test";
const tokens3 = TokenizerFSA.tokenize(text3);
console.log(tokens3);
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-02-11 10:07:25 +0000