Do not emit EOF token in tokenizer

Instead, pretend that one had been emitted after the main constructTree
loop. (This way we can get rid of an unnecessary test.)

2 files changed, 4 insertions, 4 deletions

diff --git a/chame/htmlparser.nim b/chame/htmlparser.nim
index 4da490de..defa3eb3 100644
--- a/chame/htmlparser.nim
+++ b/chame/htmlparser.nim
@@ -2917,8 +2917,7 @@ proc constructTree[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) =
           else:
             token.s.delete(0..0)
     if parser.openElements.len == 0 or
-        parser.getNamespace(parser.adjustedCurrentNode) == Namespace.HTML or
-        token.t == EOF:
+        parser.getNamespace(parser.adjustedCurrentNode) == Namespace.HTML:
       parser.processInHTMLContent(token, parser.insertionMode)
     else:
       let oe = parser.adjustedCurrentNodeToken
@@ -2942,7 +2941,8 @@ proc constructTree[Handle, Atom](parser: var HTML5Parser[Handle, Atom]) =
       else:
         parser.processInForeignContent(token)
     if parser.stopped:
-      break
+      return
+  parser.processInHTMLContent(Token[Atom](t: EOF), parser.insertionMode)
 
 proc finishParsing(parser: var HTML5Parser) =
   while parser.openElements.len > 0:
diff --git a/chame/htmltokenizer.nim b/chame/htmltokenizer.nim
index 34f52f45..1d56c918 100644
--- a/chame/htmltokenizer.nim
+++ b/chame/htmltokenizer.nim
@@ -326,7 +326,7 @@ iterator tokenize*[Atom](tokenizer: var Tokenizer[Atom]): Token[Atom] =
     tokenizer.flushChars()
     emit Token[Atom](t: CHARACTER_NULL)
   template emit_eof =
-    emit EOF
+    tokenizer.flushChars()
     running = false
   template emit_tok =
     #TODO this should only be checked for cases where a start tag could