From 0a27cca4b50e73beff5dbef84e6de6ab9a4f9f21 Mon Sep 17 00:00:00 2001 From: Timothee Cour Date: Mon, 1 Jun 2020 06:56:29 -0700 Subject: runnableExamples: correctly handle multiline string litterals (#14492) * runnableExamples: correctly handle multiline string litterals * address comments: improve doc comments + variable namings --- compiler/renderverbatim.nim | 83 ++++++++++++++++++++++++++-- nimdoc/testproject/expected/testproject.html | 45 +++++++++++++++ nimdoc/testproject/expected/testproject.idx | 1 + nimdoc/testproject/expected/theindex.html | 4 ++ nimdoc/testproject/testproject.nim | 40 ++++++++++++++ 5 files changed, 168 insertions(+), 5 deletions(-) diff --git a/compiler/renderverbatim.nim b/compiler/renderverbatim.nim index 974454456..d20ee1549 100644 --- a/compiler/renderverbatim.nim +++ b/compiler/renderverbatim.nim @@ -4,8 +4,10 @@ from xmltree import addEscaped import ast, options, msgs import packages/docutils/highlite -# import compiler/renderer -import renderer +const isDebug = false +when isDebug: + import renderer + import astalgo proc lastNodeRec(n: PNode): PNode = result = n @@ -23,6 +25,66 @@ proc isInIndentationBlock(src: string, indent: int): bool = if src[j] != ' ': return false return true +type LineData = object + ## keep track of which lines are starting inside a multiline doc comment. + ## We purposefully avoid re-doing parsing which is already done (we get a PNode) + ## so we don't worry about whether we're inside (nested) doc comments etc. + ## But we sill need some logic to disambiguate different multiline styles. + conf: ConfigRef + lineFirst: int + lines: seq[bool] + ## lines[index] is true if line `lineFirst+index` starts inside a multiline string + ## Using a HashSet (extra dependency) would simplify but not by much. + +proc tripleStrLitStartsAtNextLine(conf: ConfigRef, n: PNode): bool = + # enabling TLineInfo.offsetA,offsetB would probably make this easier + const tripleQuote = "\"\"\"" + let src = sourceLine(conf, n.info) + let col = n.info.col + doAssert src.continuesWith(tripleQuote, col) # sanity check + var i = col + 3 + var onlySpace = true + while true: + if src.len <= i: + doAssert src.len == i + return onlySpace + elif src.continuesWith(tripleQuote, i) and (src.len == i+3 or src[i+3] != '\"'): + return false # triple lit is in 1 line + elif src[i] != ' ': onlySpace = false + i.inc + +proc visitMultilineStrings(ldata: var LineData, n: PNode) = + var cline = ldata.lineFirst + + template setLine() = + let index = cline - ldata.lineFirst + if ldata.lines.len < index+1: ldata.lines.setLen index+1 + ldata.lines[index] = true + + case n.kind + of nkTripleStrLit: + # same logic should be applied for any multiline token + # we could also consider nkCommentStmt but right now we just assume doc comments, + # unlike triple string litterals, don't de-indent from runnableExamples. + cline = n.info.line.int + if tripleStrLitStartsAtNextLine(ldata.conf, n): + cline.inc + setLine() + for ai in n.strVal: + case ai + of '\n': + cline.inc + setLine() + else: discard + else: + for i in 0..= ldata.lines.len: false + else: ldata.lines[index] + proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string = ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty, ## we'd need to check performance impact to enable it for nimdoc. @@ -39,20 +101,31 @@ proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string = assert true ]# first.line = n[0].info.line + 1 - # first.col = n[0].info.col + 1 # anything with `col > n[0].col` is part of runnableExamples let last = n.lastNodeRec.info var info = first var indent = info.col let numLines = numLines(conf, info.fileIndex).uint16 var lastNonemptyPos = 0 + + var ldata = LineData(lineFirst: first.line.int, conf: conf) + visitMultilineStrings(ldata, n[^1]) + when isDebug: + debug(n) + for i in 0.. last.line and not isInIndentationBlock(src, indent): + let special = startOfLineInsideTriple(ldata, line.int) + if line > last.line and not special and not isInIndentationBlock(src, indent): break if line > first.line: result.add "\n" - if src.len > indent: + if special: + result.add src + lastNonemptyPos = result.len + elif src.len > indent: result.add src[indent..^1] lastNonemptyPos = result.len result.setLen lastNonemptyPos diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html index 4fd5d8ad3..7bd2e64a3 100644 --- a/nimdoc/testproject/expected/testproject.html +++ b/nimdoc/testproject/expected/testproject.html @@ -198,6 +198,8 @@ function main() { title="low[T: Ordinal | enum | range](x: T): T">low
  • low2
    • +
  • tripleStrLitTest
    • @@ -594,6 +596,49 @@ the c printf. etc.

    Example:

    discard "in low2"
    + + +
    proc tripleStrLitTest() {...}{.raises: [], tags: [].}
    +
    + + +

    Example:

    +
    ## mullitline string litterals are tricky as their indentation can span
+## below that of the runnableExamples
+let s1a = """
+should appear at indent 0
+  at indent 2
+at indent 0
+"""
+# make sure this works too
+let s1b = """start at same line
+  at indent 2
+at indent 0
+""" # comment after
+let s2 = """sandwich """
+let s3 = """"""
+when false:
+  let s5 = """
+        in s5 """
+
+let s3b = ["""
+%!? #[...] # inside a multiline ...
+""", "foo"]
+
+## make sure handles trailing spaces
+let s4 = """ 
+"""
+
+let s5 = """ x
+"""
+let s6 = """ ""
+"""
+let s7 = """"""""""
+let s8 = ["""""""""", """
+  """ ]
+discard
+# should be in
    +
    diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx index dd26c03d9..46798f314 100644 --- a/nimdoc/testproject/expected/testproject.idx +++ b/nimdoc/testproject/expected/testproject.idx @@ -37,6 +37,7 @@ c_printf testproject.html#c_printf,cstring testproject: c_printf(frmt: cstring): c_nonexistant testproject.html#c_nonexistant,cstring testproject: c_nonexistant(frmt: cstring): cint low testproject.html#low,T testproject: low[T: Ordinal | enum | range](x: T): T low2 testproject.html#low2,T testproject: low2[T: Ordinal | enum | range](x: T): T +tripleStrLitTest testproject.html#tripleStrLitTest testproject: tripleStrLitTest() bar testproject.html#bar.m testproject: bar(): untyped z16 testproject.html#z16.m testproject: z16() z18 testproject.html#z18.m testproject: z18(): int diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html index 1e9e4374f..14d07bfd9 100644 --- a/nimdoc/testproject/expected/theindex.html +++ b/nimdoc/testproject/expected/theindex.html @@ -213,6 +213,10 @@ function main() {
  • testproject: testNimDocTrailingExample()
    • +
    tripleStrLitTest:
    z1:
    • testproject: z1(): Foo
      • diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim index 45bbca223..b05754d51 100644 --- a/nimdoc/testproject/testproject.nim +++ b/nimdoc/testproject/testproject.nim @@ -207,6 +207,46 @@ when true: # tests RST inside comments runnableExamples: discard "in low2" +when true: # multiline string litterals + proc tripleStrLitTest*() = + runnableExamples: + ## mullitline string litterals are tricky as their indentation can span + ## below that of the runnableExamples + let s1a = """ +should appear at indent 0 + at indent 2 +at indent 0 +""" + # make sure this works too + let s1b = """start at same line + at indent 2 +at indent 0 +""" # comment after + let s2 = """sandwich """ + let s3 = """""" + when false: + let s5 = """ + in s5 """ + + let s3b = [""" +%!? #[...] # inside a multiline ... +""", "foo"] + + ## make sure handles trailing spaces + let s4 = """ +""" + + let s5 = """ x +""" + let s6 = """ "" +""" + let s7 = """""""""" + let s8 = ["""""""""", """ + """ ] + discard + # should be in + # should be out + when true: # (most) macros macro bar*(): untyped = result = newStmtList() -- cgit 1.4.1-2-gfad0