diff options
| -rw-r--r-- | lib/packages/docutils/highlite.nim | 3 | ||||
| -rw-r--r-- | tests/stdlib/thighlite.nim | 10 |
2 files changed, 13 insertions, 0 deletions
diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim index 3f65c8cc4..c3caf31b4 100644 --- a/lib/packages/docutils/highlite.nim +++ b/lib/packages/docutils/highlite.nim @@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string], of '\0': break else: inc(pos) + else: + g.kind = gtOperator + while g.buf[pos] in OpChars: inc(pos) of '#': inc(pos) if hasPreprocessor in flags: diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim index 5134215c1..d88bc7fec 100644 --- a/tests/stdlib/thighlite.nim +++ b/tests/stdlib/thighlite.nim @@ -1,6 +1,7 @@ import unittest, strutils import ../../lib/packages/docutils/highlite +import std/objectdollar block: # Nim tokenizing test "string literals and escape seq": @@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing ("file.nim", gtIdentifier), ("\n", gtWhitespace), ("out: file [SuccessX]", gtProgramOutput) ]) + +block: # bug #21232 + let code = "/" + var toknizr: GeneralTokenizer + + initGeneralTokenizer(toknizr, code) + + getNextToken(toknizr, langC) + check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)""" |