fixes #21232; highlite.getNextToken() returns gtOperator for '/' in C (#21234)

2 files changed, 13 insertions, 0 deletions

diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim
index 3f65c8cc4..c3caf31b4 100644
--- a/lib/packages/docutils/highlite.nim
+++ b/lib/packages/docutils/highlite.nim
@@ -498,6 +498,9 @@ proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
           of '\0':
             break
           else: inc(pos)
+      else:
+        g.kind = gtOperator
+        while g.buf[pos] in OpChars: inc(pos)
     of '#':
       inc(pos)
       if hasPreprocessor in flags:
diff --git a/tests/stdlib/thighlite.nim b/tests/stdlib/thighlite.nim
index 5134215c1..d88bc7fec 100644
--- a/tests/stdlib/thighlite.nim
+++ b/tests/stdlib/thighlite.nim
@@ -1,6 +1,7 @@
 
 import unittest, strutils
 import ../../lib/packages/docutils/highlite
+import std/objectdollar
 
 block: # Nim tokenizing
   test "string literals and escape seq":
@@ -30,3 +31,12 @@ block: # Cmd (shell) tokenizing
         ("file.nim", gtIdentifier), ("\n", gtWhitespace),
         ("out: file [SuccessX]", gtProgramOutput)
       ])
+
+block: # bug #21232
+  let code = "/"
+  var toknizr: GeneralTokenizer
+
+  initGeneralTokenizer(toknizr, code)
+
+  getNextToken(toknizr, langC)
+  check $toknizr == """(kind: gtOperator, start: 0, length: 1, buf: "/", pos: 1, state: gtEof, lang: langC)"""