changed comment handling (breaking change)

3 files changed, 49 insertions, 80 deletions

diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index c2587efad..5e15b5c5e 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -482,7 +482,7 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     add(tok.literal, VT)
     inc(L.bufpos)
   of 't', 'T': 
-    add(tok.literal, Tabulator)
+    add(tok.literal, '\t')
     inc(L.bufpos)
   of '\'', '\"': 
     add(tok.literal, L.buf[L.bufpos])
@@ -659,28 +659,29 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
-proc scanComment(L: var TLexer, tok: var TToken) = 
+proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
-  var buf = L.buf 
-  # a comment ends if the next line does not start with the # on the same
-  # column after only whitespace
+  when not defined(nimfix): assert buf[pos+1] == '#'
+  var buf = L.buf
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
-  var col = getColNumber(L, pos)
+  when defined(nimfix):
+    var col = getColNumber(L, pos)
   while true:
     var lastBackslash = -1
     while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
       if buf[pos] == '\\': lastBackslash = pos+1
       add(tok.literal, buf[pos])
       inc(pos)
-    if lastBackslash > 0:
-      # a backslash is a continuation character if only followed by spaces
-      # plus a newline:
-      while buf[lastBackslash] == ' ': inc(lastBackslash)
-      if buf[lastBackslash] notin {CR, LF, nimlexbase.EndOfFile}:
-        # false positive:
-        lastBackslash = -1
+    when defined(nimfix):
+      if lastBackslash > 0:
+        # a backslash is a continuation character if only followed by spaces
+        # plus a newline:
+        while buf[lastBackslash] == ' ': inc(lastBackslash)
+        if buf[lastBackslash] notin {CR, LF, nimlexbase.EndOfFile}:
+          # false positive:
+          lastBackslash = -1
 
     pos = handleCRLF(L, pos)
     buf = L.buf
@@ -688,9 +689,16 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     while buf[pos] == ' ': 
       inc(pos)
       inc(indent)
-    if buf[pos] == '#' and (col == indent or lastBackslash > 0):
+
+    when defined(nimfix):
+      template doContinue(): expr =
+        buf[pos] == '#' and (col == indent or lastBackslash > 0)
+    else:
+      template doContinue(): expr =
+        buf[pos] == '#' and buf[pos+1] == '#'
+    if doContinue():
       tok.literal.add "\n"
-      col = indent
+      when defined(nimfix): col = indent
       inc tok.iNumber
     else:
       if buf[pos] > ' ': 
@@ -707,7 +715,7 @@ proc skip(L: var TLexer, tok: var TToken) =
     of ' ':
       inc(pos)
       inc(tok.strongSpaceA)
-    of Tabulator:
+    of '\t':
       lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
       inc(pos)
     of CR, LF:
@@ -718,7 +726,12 @@ proc skip(L: var TLexer, tok: var TToken) =
         inc(pos)
         inc(indent)
       tok.strongSpaceA = 0
-      if buf[pos] > ' ':
+      when defined(nimfix):
+        template doBreak(): expr = buf[pos] > ' '
+      else:
+        template doBreak(): expr =
+          buf[pos] > ' ' and (buf[pos] != '#' or buf[pos+1] == '#')
+      if doBreak():
         tok.indent = indent
         L.currLineIndent = indent
         break
diff --git a/doc/manual.txt b/doc/manual.txt
index 144a3b2c0..f0f50e373 100644
--- a/doc/manual.txt
+++ b/doc/manual.txt
@@ -156,33 +156,20 @@ Comments start anywhere outside a string or character literal with the
 hash character ``#``.
 Comments consist of a concatenation of `comment pieces`:idx:. A comment piece
 starts with ``#`` and runs until the end of the line. The end of line characters
-belong to the piece. If the next line only consists of a comment piece which is
-aligned to the preceding one, it does not start a new comment:
-
-.. code-block:: nim
-
-  i = 0     # This is a single comment over multiple lines belonging to the
-            # assignment statement. The scanner merges these two pieces.
-  # This is a new comment belonging to the current block, but to no particular
-  # statement.
-  i = i + 1 # This a new comment that is NOT
-  echo(i)   # continued here, because this comment refers to the echo statement
+belong to the piece. If the next line only consists of a comment piece with
+no other tokens between it and the preceding one, it does not start a new
+comment:
 
 
-The alignment requirement does not hold if the preceding comment piece ends in
-a backslash (followed by optional whitespace): 
-
 .. code-block:: nim
-  type
-    TMyObject {.final, pure, acyclic.} = object  # comment continues: \
-      # we have lots of space here to comment 'TMyObject'.
-      # This line belongs to the comment as it's properly aligned.
+  i = 0     # This is a single comment over multiple lines.
+    # The scanner merges these two pieces.
+    # The comment continues here.
+
 
-Comments are tokens; they are only allowed at certain places in the input file
-as they belong to the syntax tree! This feature enables perfect source-to-source
-transformations (such as pretty-printing) and superior documentation generators.
-A nice side-effect is that the human reader of the code always knows exactly
-which code snippet the comment refers to.
+`Documentation comments`:idx: are comments that start with two ``##``.
+Documentation comments are tokens; they are only allowed at certain places in
+the input file as they belong to the syntax tree!
 
 
 Identifiers & Keywords
@@ -5804,7 +5791,7 @@ Spawn
 
 Nim has a builtin thread pool that can be used for CPU intensive tasks. For
 IO intensive tasks the upcoming ``async`` and ``await`` features should be
-used instead. `spawn`:idx: is used to pass a task to the thread pool:
+used. `spawn`:idx: is used to pass a task to the thread pool:
 
 .. code-block:: nim
   proc processLine(line: string) =
diff --git a/doc/tut1.txt b/doc/tut1.txt
index 73d90b008..b90736bd0 100644
--- a/doc/tut1.txt
+++ b/doc/tut1.txt
@@ -112,50 +112,19 @@ Comments
 --------
 
 Comments start anywhere outside a string or character literal with the
-hash character ``#``. Documentation comments start with ``##``. Multiline
-comments need to be aligned at the same column:
+hash character ``#``. Documentation comments start with ``##``:
 
 .. code-block:: nim
+  # A comment.
+ 
+  var myVariable: int ## a documentation comment
 
-  i = 0     # This is a single comment over multiple lines belonging to the
-            # assignment statement.
-  # This is a new comment belonging to the current block, but to no particular
-  # statement.
-  i = i + 1 # This a new comment that is NOT
-  echo(i)   # continued here, because this comment refers to the echo statement
 
+Documentation comments are tokens; they are only allowed at certain places in
+the input file as they belong to the syntax tree! This feature enables simpler
+documentation generators.
 
-The alignment requirement does not hold if the preceding comment piece ends in
-a backslash:
-
-.. code-block:: nim
-  type
-    TMyObject {.final, pure, acyclic.} = object  # comment continues: \
-      # we have lots of space here to comment 'TMyObject'.
-      # This line belongs to the comment as it's properly aligned.
-
-
-Comments are tokens; they are only allowed at certain places in the input file
-as they belong to the syntax tree! This feature enables perfect source-to-source
-transformations (such as pretty-printing) and simpler documentation generators.
-A nice side-effect is that the human reader of the code always knows exactly
-which code snippet the comment refers to. Since comments are a proper part of
-the syntax, watch their indentation:
-
-.. code-block::
-  echo("Hello!")
-  # comment has the same indentation as above statement -> fine
-  echo("Hi!")
-    # comment has not the correct indentation level -> syntax error!
-
-**Note**: To comment out a large piece of code, it is often better to use a
-``when false:`` statement.
-
-.. code-block:: nim
-  when false:
-    brokenCode()
-
-Another option is to use the `discard statement`_ together with *long string
+You can also use the `discard statement`_ together with *long string
 literals* to create block comments:
 
 .. code-block:: nim