Merge branch 'multi-line-comments' into devel

8 files changed, 158 insertions, 38 deletions

diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index 952366f93..8555ec4f0 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -149,10 +149,10 @@ proc ropeFormatNamedVars(frmt: FormatStr, varnames: openArray[string],
 proc genComment(d: PDoc, n: PNode): string =
   result = ""
   var dummyHasToc: bool
-  if n.comment != nil and startsWith(n.comment, "##"):
+  if n.comment != nil:
     renderRstToOut(d[], parseRst(n.comment, toFilename(n.info),
                                toLinenumber(n.info), toColumn(n.info),
-                               dummyHasToc, d.options + {roSkipPounds}), result)
+                               dummyHasToc, d.options), result)
 
 proc genRecComment(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 9a69ede3e..4c1dd366b 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -769,24 +769,88 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
+proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+                          isDoc: bool) =
+  var pos = start
+  var buf = L.buf
+  var toStrip = 0
+  # detect the amount of indentation:
+  if isDoc:
+    toStrip = getColNumber(L, pos)
+    while buf[pos] == ' ': inc pos
+    if buf[pos] in {CR, LF}:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      toStrip = 0
+      while buf[pos] == ' ':
+        inc pos
+        inc toStrip
+  var nesting = 0
+  while true:
+    case buf[pos]
+    of '#':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '[':
+          inc nesting
+        tok.literal.add '#'
+      elif buf[pos+1] == '[':
+        inc nesting
+      inc pos
+    of ']':
+      if isDoc:
+        if buf[pos+1] == '#' and buf[pos+2] == '#':
+          if nesting == 0:
+            inc(pos, 3)
+            break
+          dec nesting
+        tok.literal.add ']'
+      elif buf[pos+1] == '#':
+        if nesting == 0:
+          inc(pos, 2)
+          break
+        dec nesting
+      inc pos
+    of '\t':
+      lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
+      inc(pos)
+      if isDoc: tok.literal.add '\t'
+    of CR, LF:
+      pos = handleCRLF(L, pos)
+      buf = L.buf
+      # strip leading whitespace:
+      if isDoc:
+        tok.literal.add "\n"
+        inc tok.iNumber
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
+    of nimlexbase.EndOfFile:
+      lexMessagePos(L, errGenerated, pos, "end of multiline comment expected")
+      break
+    else:
+      if isDoc: tok.literal.add buf[pos]
+      inc(pos)
+  L.bufpos = pos
+
 proc scanComment(L: var TLexer, tok: var TToken) =
   var pos = L.bufpos
   var buf = L.buf
+  tok.tokType = tkComment
+  # iNumber contains the number of '\n' in the token
+  tok.iNumber = 0
   when not defined(nimfix):
     assert buf[pos+1] == '#'
     if buf[pos+2] == '[':
-      if buf[pos+3] == ']':
-        #  ##[] is the (rather complex) "cursor token" for idetools
-        tok.tokType = tkComment
-        tok.literal = "[]"
-        inc(L.bufpos, 4)
-        return
-      else:
-        lexMessagePos(L, warnDeprecated, pos, "use '## [' instead; '##['")
+      skipMultiLineComment(L, tok, pos+3, true)
+      return
+    inc(pos, 2)
+
+  var toStrip = 0
+  while buf[pos] == ' ':
+    inc pos
+    inc toStrip
 
-  tok.tokType = tkComment
-  # iNumber contains the number of '\n' in the token
-  tok.iNumber = 0
   when defined(nimfix):
     var col = getColNumber(L, pos)
   while true:
@@ -820,6 +884,12 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     if doContinue():
       tok.literal.add "\n"
       when defined(nimfix): col = indent
+      else:
+        inc(pos, 2)
+        var c = toStrip
+        while buf[pos] == ' ' and c > 0:
+          inc pos
+          dec c
       inc tok.iNumber
     else:
       if buf[pos] > ' ':
@@ -863,7 +933,8 @@ proc skip(L: var TLexer, tok: var TToken) =
         # do not skip documentation comment:
         if buf[pos+1] == '#': break
         if buf[pos+1] == '[':
-          lexMessagePos(L, warnDeprecated, pos, "use '# [' instead; '#['")
+          skipMultiLineComment(L, tok, pos+2, false)
+          return
         while buf[pos] notin {CR, LF, nimlexbase.EndOfFile}: inc(pos)
     else:
       break                   # EndOfFile also leaves the loop
diff --git a/compiler/parser.nim b/compiler/parser.nim
index eb0ef071d..c4681a5cd 100644
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -112,12 +112,7 @@ proc rawSkipComment(p: var TParser, node: PNode) =
   if p.tok.tokType == tkComment:
     if node != nil:
       if node.comment == nil: node.comment = ""
-      if p.tok.literal == "[]":
-        node.flags.incl nfIsCursor
-        #echo "parser: "
-        #debug node
-      else:
-        add(node.comment, p.tok.literal)
+      add(node.comment, p.tok.literal)
     else:
       parMessage(p, errInternal, "skipComment")
     getTok(p)
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index c4d1222c4..8e4aa1831 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -167,33 +167,24 @@ proc makeNimString(s: string): string =
 proc putComment(g: var TSrcGen, s: string) =
   if s.isNil: return
   var i = 0
-  var comIndent = 1
   var isCode = (len(s) >= 2) and (s[1] != ' ')
   var ind = g.lineLen
-  var com = ""
+  var com = "## "
   while true:
     case s[i]
     of '\0':
       break
     of '\x0D':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       if s[i] == '\x0A': inc(i)
       optNL(g, ind)
     of '\x0A':
       put(g, tkComment, com)
-      com = ""
+      com = "## "
       inc(i)
       optNL(g, ind)
-    of '#':
-      add(com, s[i])
-      inc(i)
-      comIndent = 0
-      while s[i] == ' ':
-        add(com, s[i])
-        inc(i)
-        inc(comIndent)
     of ' ', '\x09':
       add(com, s[i])
       inc(i)
@@ -206,7 +197,7 @@ proc putComment(g: var TSrcGen, s: string) =
       if not isCode and (g.lineLen + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
-        com = '#' & spaces(comIndent)
+        com = "## "
       while s[i] > ' ':
         add(com, s[i])
         inc(i)
@@ -283,7 +274,7 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
   result = false
   if n.comment != nil:
     result = (renderNoComments notin g.flags) or
-        (renderDocComments in g.flags) and startsWith(n.comment, "##")
+        (renderDocComments in g.flags)
 
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
diff --git a/doc/manual/lexing.txt b/doc/manual/lexing.txt
index 7f81ab422..7655964ee 100644
--- a/doc/manual/lexing.txt
+++ b/doc/manual/lexing.txt
@@ -69,6 +69,35 @@ Documentation comments are tokens; they are only allowed at certain places in
 the input file as they belong to the syntax tree!
 
 
+Multiline comments
+------------------
+
+Starting with version 0.13.0 of the language Nim supports multiline comments.
+They look like:
+
+.. code-block:: nim
+  #[Comment here.
+  Multiple lines
+  are not a problem.]#
+
+Multiline comments support nesting:
+
+.. code-block:: nim
+  # Does not comment out 'p' properly:
+  #[  #[ Multiline comment in already
+     commented out code. ]#
+  proc p[T](x: T) = discard
+  ]#
+
+Multiline documentation comments look like and support nesting too:
+
+.. code-block:: nim
+  proc foo =
+    ##[Long documentation comment
+    here.
+    ]##
+
+
 Identifiers & Keywords
 ----------------------
 
diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim
index 640b8cd5a..1bc0af1b6 100644
--- a/lib/packages/docutils/highlite.nim
+++ b/lib/packages/docutils/highlite.nim
@@ -173,7 +173,41 @@ proc nimNextToken(g: var GeneralTokenizer) =
       while g.buf[pos] in {' ', '\x09'..'\x0D'}: inc(pos)
     of '#':
       g.kind = gtComment
-      while not (g.buf[pos] in {'\0', '\x0A', '\x0D'}): inc(pos)
+      inc(pos)
+      var isDoc = false
+      if g.buf[pos] == '#':
+        inc(pos)
+        isDoc = true
+      if g.buf[pos] == '[':
+        g.kind = gtLongComment
+        var nesting = 0
+        while true:
+          case g.buf[pos]
+          of '\0': break
+          of '#':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '[':
+                inc nesting
+            elif g.buf[pos+1] == '[':
+              inc nesting
+            inc pos
+          of ']':
+            if isDoc:
+              if g.buf[pos+1] == '#' and g.buf[pos+2] == '#':
+                if nesting == 0:
+                  inc(pos, 3)
+                  break
+                dec nesting
+            elif g.buf[pos+1] == '#':
+              if nesting == 0:
+                inc(pos, 2)
+                break
+              dec nesting
+            inc pos
+          else:
+            inc pos
+      else:
+        while g.buf[pos] notin {'\0', '\x0A', '\x0D'}: inc(pos)
     of 'a'..'z', 'A'..'Z', '_', '\x80'..'\xFF':
       var id = ""
       while g.buf[pos] in SymChars + {'_'}:
diff --git a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim
index e64b818fa..142b190ab 100644
--- a/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim
+++ b/tests/manyloc/keineschweine/dependencies/genpacket/genpacket_enet.nim
@@ -131,10 +131,10 @@ macro defPacket*(typeNameN: expr, typeFields: expr): stmt {.immediate.} =
           emptyNode())),
       emptyNode(),
       emptyNode(),
-      newNimNode(nnkStmtList).und(#[6]
+      newNimNode(nnkStmtList).und(# [6]
         newNimNode(nnkAsgn).und(
           ^"result",                  ## result =
-          newNimNode(nnkCall).und(#[6][0][1]
+          newNimNode(nnkCall).und(# [6][0][1]
             ^"format",  ## format
             emptyNode()))))  ## "[TypeName   $1   $2]"
     formatStr = "["& $typeName.ident
diff --git a/todo.txt b/todo.txt
index ca8d60762..6c1c602f5 100644
--- a/todo.txt
+++ b/todo.txt
@@ -10,7 +10,6 @@ essential for 1.0
 - make '--implicitStatic:on' the default; then we can also clean up the
   'static[T]' mess in the compiler!
 
-- add "all threads are blocked" detection to 'spawn'
 - Deprecate ``immediate`` for templates and macros
 - document NimMain and check whether it works for threading
 - ``not`` or ``~`` for the effects system
@@ -19,6 +18,7 @@ essential for 1.0
 Not critical for 1.0
 ====================
 
+- add "all threads are blocked" detection to 'spawn'
 - figure out why C++ bootstrapping is so much slower
 - The bitwise 'not' operator cold be renamed to 'bnot' to
   prevent 'not 4 == 5' from compiling. -> requires 'mixin' annotation for procs!