fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234)

1 files changed, 10 insertions, 0 deletions

diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim
index 5134215c1..d88bc7fec 100644
--- a/tests/stdlib/thighlite.nim
+++ b/tests/stdlib/thighlite.nim
@@ -1,6 +1,7 @@
 
 import unittest, strutils
 import ../../lib/packages/docutils/highlite
+import std/objectdollar
 
 block: # Nim tokenizing
   test "string literals and escape seq":
@@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing
         ("file.nim", gtIdentifier), ("\n", gtWhitespace),
         ("out: file [SuccessX]", gtProgramOutput)
       ])
+
+block: # bug #21232
+  let code = "/"
+  var toknizr: GeneralTokenizer
+
+  initGeneralTokenizer(toknizr, code)
+
+  getNextToken(toknizr, langC)
+  check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)"""