fixes the bug that keeps the template engine package from working

3 files changed, 677 insertions, 0 deletions

diff --git a/tests/template/annotate.nim b/tests/template/annotate.nim
new file mode 100644
index 000000000..fa58030dc
--- /dev/null
+++ b/tests/template/annotate.nim
@@ -0,0 +1,113 @@
+import macros, parseutils
+
+# Generate tags
+macro make(names: openarray[expr]): stmt {.immediate.} =
+    result = newStmtList()
+
+    for i in 0 .. names.len-1:
+        result.add newProc(
+            name   = ident($names[i]).postfix("*"),
+            params = [
+                ident("string"),
+                newIdentDefs(
+                    ident("content"),
+                    ident("string")
+                )
+            ],
+            body = newStmtList(
+                parseStmt("reindent(content)")
+            )
+        )
+
+
+iterator lines(value: string): string =
+    var i = 0
+    while i < value.len:
+        var line: string
+        inc(i, value.parseUntil(line, 0x0A.char, i) + 1)
+        yield line
+
+
+proc reindent*(value: string, preset_indent = 0): string =
+    var indent = -1
+
+    # Detect indentation!
+    for ln in lines(value):
+        var read = ln.skipWhitespace()
+
+        # If the line is empty, ignore it for indentation
+        if read == ln.len: continue
+
+        indent = if indent < 0: read
+                 else: min(indent, read)
+
+    # Create a precursor indent as-needed
+    var precursor = newString(0)
+    for i in 1 .. preset_indent:
+        precursor.add(' ')
+
+    # Re-indent
+    result = newString(0)
+
+    for ln in lines(value):
+        var value = ln.substr(indent)
+
+        result.add(precursor)
+
+        if value.len > 0:
+            result.add(value)
+            result.add(0x0A.char)
+
+    return result
+
+
+#Define tags
+make([ html, xml, glsl, js, css, rst ])
+
+
+when isMainModule:
+    ## Test tags
+
+    const script = js"""
+        var x = 5;
+        console.log(x.toString());
+    """
+
+    const styles = css"""
+        .someRule {
+            width: 500px;
+        }
+    """
+
+    const body = html"""
+        <ul>
+            <li>1</li>
+            <li>2</li>
+            <li>
+                <a hef="#google">google</a>
+            </li>
+        </ul>
+    """
+
+    const info = xml"""
+        <item>
+            <i>1</i>
+            <i>2</i>
+        </item>
+    """
+
+    const shader = glsl"""
+        void main()
+        {
+            gl_Position = gl_ProjectionMatrix
+                        * gl_ModelViewMatrix
+                        * gl_Vertex;
+        }
+    """
+
+
+    echo script
+    echo styles
+    echo body
+    echo info
+    echo shader
\ No newline at end of file
diff --git a/tests/template/otests.nim b/tests/template/otests.nim
new file mode 100644
index 000000000..c885e23df
--- /dev/null
+++ b/tests/template/otests.nim
@@ -0,0 +1,219 @@
+# Fields

+const x = 5

+

+

+# Test substring

+static:

+    assert "test".substring(3)   == "t"

+    assert "test".substring(2,1) == "s"

+    assert "test".substring(3,2) == "t"

+    assert "test".substring(1,2) == "es"

+

+

+# Various parsing tests

+when true:

+

+    block: #no_substitution

+        proc actual: string = tmpli html"""

+            <p>Test!</p>

+        """

+        const expected = html"""

+            <p>Test!</p>

+        """

+        doAssert actual() == expected

+

+    block: #basic

+        proc actual: string = tmpli html"""

+            <p>Test $$x</p>

+            $x

+        """

+        const expected = html"""

+            <p>Test $x</p>

+            5

+        """

+        doAssert actual() == expected

+

+    block: #expression

+        proc actual: string = tmpli html"""

+            <p>Test $$(x * 5)</p>

+            $(x * 5)

+        """

+        const expected = html"""

+            <p>Test $(x * 5)</p>

+            25

+        """

+        doAssert actual() == expected

+

+    block: #escape

+        proc actual: string = tmpli js"""

+            [{

+                "hello world"

+            }]

+        """

+        const expected = js"""

+            [{

+                "hello world"

+            }]

+        """

+        doAssert actual() == expected

+

+    block: #forIn

+        proc actual: string = tmpli html"""

+            <p>Test for</p>

+            <ul>

+                $for y in 0..2 {

+                    <li>$y</li>

+                }

+            </ul>

+        """

+        const expected = html"""

+            <p>Test for</p>

+            <ul>

+                <li>0</li>

+                <li>1</li>

+                <li>2</li>

+            </ul>

+        """

+        doAssert actual() == expected

+

+    block: #while

+        proc actual: string = tmpli html"""

+            <p>Test while/stmt</p>

+            <ul>

+                ${ var y = 0 }

+                $while y < 4 {

+                    <li>$y</li>

+                    ${ inc(y) }

+                }

+            </ul>

+        """

+        const expected = html"""

+            <p>Test while/stmt</p>

+            <ul>

+                <li>0</li>

+                <li>1</li>

+                <li>2</li>

+                <li>3</li>

+            </ul>

+        """

+        doAssert actual() == expected

+

+    block: #ifElifElse

+        proc actual: string = tmpli html"""

+            <p>Test if/elif/else</p>

+            $if x == 8 {

+                <div>x is 8!</div>

+            }

+            $elif x == 7 {

+                <div>x is 7!</div>

+            }

+            $else {

+                <div>x is neither!</div>

+            }

+        """

+        const expected = html"""

+            <p>Test if/elif/else</p>

+            <div>x is neither!</div>

+        """

+        doAssert actual() == expected

+

+    block: #multiLineStatements

+        proc actual: string = tmpli html"""

+            <p>Test multiline statements</p>

+            ${

+                var x = 5

+                var y = 7

+            }

+            <span>$x</span><span>$y</span>

+        """

+        const expected = html"""

+            <p>Test multiline statements</p>

+            <span>5</span><span>7</span>

+        """

+        doAssert actual() == expected

+

+    block: #caseOfElse

+        proc actual: string = tmpli html"""

+            <p>Test case</p>

+            $case x

+            $of 5 {

+                <div>x == 5</div>

+            }

+            $of 6 {

+                <div>x == 6</div>

+            }

+            $else {}

+        """

+        const expected = html"""

+            <p>Test case</p>

+            <div>x == 5</div>

+        """

+        doAssert actual() == expected

+

+

+

+when true: #embeddingTest

+    proc no_substitution: string = tmpli html"""

+        <h1>Template test!</h1>

+    """

+

+    # # Single variable substitution

+    proc substitution(who = "nobody"): string = tmpli html"""

+        <div id="greeting">hello $who!</div>

+    """

+

+    # Expression template

+    proc test_expression(nums: openarray[int] = []): string =

+        var i = 2

+        tmpli html"""

+            $(no_substitution())

+            $(substitution("Billy"))

+            <div id="age">Age: $($nums[i] & "!!")</div>

+        """

+

+    proc test_statements(nums: openarray[int] = []): string =

+        tmpli html"""

+            $(test_expression(nums))

+            $if true {

+                <ul>

+                    $for i in nums {

+                        <li>$(i * 2)</li>

+                    }

+                </ul>

+            }

+        """

+

+    var actual = test_statements([0,1,2])

+    const expected = html"""

+        <h1>Template test!</h1>

+        <div id="greeting">hello Billy!</div>

+        <div id="age">Age: 2!!</div>

+        <ul>

+            <li>0</li>

+            <li>2</li>

+            <li>4</li>

+        </ul>

+    """

+    doAssert actual == expected

+

+

+when defined(future):

+    block: #tryCatch

+        proc actual: string = tmpli html"""

+            <p>Test try/catch</p>

+            <div>

+                $try {

+                    <div>Lets try this!</div>

+                }

+                $except {

+                    <div>Uh oh!</div>

+                }

+            </div>

+        """

+        const expected = html"""

+            <p>Test try/catch</p>

+            <div>

+                    <div>Lets try this!</div>

+            </div>

+        """

+        doAssert actual() == expected
\ No newline at end of file
diff --git a/tests/template/t_otemplates.nim b/tests/template/t_otemplates.nim
new file mode 100644
index 000000000..7de728ab2
--- /dev/null
+++ b/tests/template/t_otemplates.nim
@@ -0,0 +1,345 @@
+discard """
+  output: "Success"
+"""
+
+# Ref:

+# http://nimrod-lang.org/macros.html

+# http://nimrod-lang.org/parseutils.html

+

+

+# Imports

+import tables, parseutils, macros, strutils

+import annotate

+export annotate

+

+

+# Fields

+const identChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'}

+

+

+# Procedure Declarations

+proc parse_template(node: PNimrodNode, value: string) {.compiletime.}

+

+

+# Procedure Definitions

+proc substring(value: string, index: int, length = -1): string {.compiletime.} =

+    ## Returns a string at most `length` characters long, starting at `index`.

+    return if length < 0:    value.substr(index)

+           elif length == 0: ""

+           else:             value.substr(index, index + length-1)

+

+

+proc parse_thru_eol(value: string, index: int): int {.compiletime.} =

+    ## Reads until and past the end of the current line, unless

+    ## a non-whitespace character is encountered first

+    var remainder: string

+    var read = value.parseUntil(remainder, {0x0A.char}, index)

+    if remainder.skipWhitespace() == read:

+        return read + 1

+

+

+proc trim_after_eol(value: var string) {.compiletime.} =

+    ## Trims any whitespace at end after \n

+    var toTrim = 0

+    for i in countdown(value.len-1, 0):

+        # If \n, return

+        if value[i] in [' ', '\t']: inc(toTrim)

+        else: break

+

+    if toTrim > 0:

+        value = value.substring(0, value.len - toTrim)

+

+

+proc trim_eol(value: var string) {.compiletime.} =

+    ## Removes everything after the last line if it contains nothing but whitespace

+    for i in countdown(value.len - 1, 0):

+        # If \n, trim and return

+        if value[i] == 0x0A.char:

+            value = value.substr(0, i)

+            break

+

+        # This is the first character

+        if i == 0:

+            value = ""

+            break

+

+        # Skip change

+        if not (value[i] in [' ', '\t']): break

+

+

+proc detect_indent(value: string, index: int): int {.compiletime.} =

+    ## Detects how indented the line at `index` is.

+    # Seek to the beginning of the line.

+    var lastChar = index

+    for i in countdown(index, 0):

+        if value[i] == 0x0A.char:

+            # if \n, return the indentation level

+            return lastChar - i

+        elif not (value[i] in [' ', '\t']):

+            # if non-whitespace char, decrement lastChar

+            dec(lastChar)

+

+

+proc parse_thru_string(value: string, i: var int, strType = '"') {.compiletime.} =

+    ## Parses until ending " or ' is reached.

+    inc(i)

+    if i < value.len-1:

+        inc(i, value.skipUntil({'\\', strType}, i))

+

+

+proc parse_to_close(value: string, index: int, open='(', close=')', opened=0): int {.compiletime.} =

+    ## Reads until all opened braces are closed

+    ## ignoring any strings "" or ''

+    var remainder   = value.substring(index)

+    var open_braces = opened

+    result = 0

+

+    while result < remainder.len:

+        var c = remainder[result]

+

+        if   c == open:  inc(open_braces)

+        elif c == close: dec(open_braces)

+        elif c == '"':   remainder.parse_thru_string(result)

+        elif c == '\'':  remainder.parse_thru_string(result, '\'')

+

+        if open_braces == 0: break

+        else: inc(result)

+

+

+iterator parse_stmt_list(value: string, index: var int): string =

+    ## Parses unguided ${..} block

+    var read        = value.parse_to_close(index, open='{', close='}')

+    var expressions = value.substring(index + 1, read - 1).split({ ';', 0x0A.char })

+

+    for expression in expressions:

+        let value = expression.strip

+        if value.len > 0:

+            yield value

+

+    #Increment index & parse thru EOL

+    inc(index, read + 1)

+    inc(index, value.parse_thru_eol(index))

+

+

+iterator parse_compound_statements(value, identifier: string, index: int): string =

+    ## Parses through several statements, i.e. if {} elif {} else {}

+    ## and returns the initialization of each as an empty statement

+    ## i.e. if x == 5 { ... } becomes if x == 5: nil.

+

+    template get_next_ident(expected): stmt =

+        var nextIdent: string

+        discard value.parseWhile(nextIdent, {'$'} + identChars, i)

+

+        var next: string

+        var read: int

+

+        if nextIdent == "case":

+            # We have to handle case a bit differently

+            read = value.parseUntil(next, '$', i)

+            inc(i, read)

+            yield next.strip(leading=false) & "\n"

+

+        else:

+            read = value.parseUntil(next, '{', i)

+

+            if nextIdent in expected:

+                inc(i, read)

+                # Parse until closing }, then skip whitespace afterwards

+                read = value.parse_to_close(i, open='{', close='}')

+                inc(i, read + 1)

+                inc(i, value.skipWhitespace(i))

+                yield next & ": nil\n"

+

+            else: break

+

+

+    var i = index

+    while true:

+        # Check if next statement would be valid, given the identifier

+        if identifier in ["if", "when"]:

+            get_next_ident([identifier, "$elif", "$else"])

+

+        elif identifier == "case":

+            get_next_ident(["case", "$of", "$elif", "$else"])

+

+        elif identifier == "try":

+            get_next_ident(["try", "$except", "$finally"])

+

+

+proc parse_complex_stmt(value, identifier: string, index: var int): PNimrodNode {.compiletime.} =

+    ## Parses if/when/try /elif /else /except /finally statements

+

+    # Build up complex statement string

+    var stmtString = newString(0)

+    var numStatements = 0

+    for statement in value.parse_compound_statements(identifier, index):

+        if statement[0] == '$': stmtString.add(statement.substr(1))

+        else:                   stmtString.add(statement)

+        inc(numStatements)

+

+    # Parse stmt string

+    result = parseExpr(stmtString)

+

+    var resultIndex = 0

+

+    # Fast forward a bit if this is a case statement

+    if identifier == "case":

+        inc(resultIndex)

+

+    while resultIndex < numStatements:

+

+        # Detect indentation

+        let indent = detect_indent(value, index)

+

+        # Parse until an open brace `{`

+        var read = value.skipUntil('{', index)

+        inc(index, read + 1)

+

+        # Parse through EOL

+        inc(index, value.parse_thru_eol(index))

+

+        # Parse through { .. }

+        read = value.parse_to_close(index, open='{', close='}', opened=1)

+

+        # Add parsed sub-expression into body

+        var body       = newStmtList()

+        var stmtString = value.substring(index, read)

+        trim_after_eol(stmtString)

+        stmtString = reindent(stmtString, indent)

+        parse_template(body, stmtString)

+        inc(index, read + 1)

+

+        # Insert body into result

+        var stmtIndex = macros.high(result[resultIndex])

+        result[resultIndex][stmtIndex] = body

+

+        # Parse through EOL again & increment result index

+        inc(index, value.parse_thru_eol(index))

+        inc(resultIndex)

+

+

+proc parse_simple_statement(value: string, index: var int): PNimrodNode {.compiletime.} =

+    ## Parses for/while

+

+    # Detect indentation

+    let indent = detect_indent(value, index)

+

+    # Parse until an open brace `{`

+    var splitValue: string

+    var read = value.parseUntil(splitValue, '{', index)

+    result   = parseExpr(splitValue & ":nil")

+    inc(index, read + 1)

+

+    # Parse through EOL

+    inc(index, value.parse_thru_eol(index))

+

+    # Parse through { .. }

+    read = value.parse_to_close(index, open='{', close='}', opened=1)

+

+    # Add parsed sub-expression into body

+    var body       = newStmtList()

+    var stmtString = value.substring(index, read)

+    trim_after_eol(stmtString)

+    stmtString = reindent(stmtString, indent)

+    parse_template(body, stmtString)

+    inc(index, read + 1)

+

+    # Insert body into result

+    var stmtIndex = macros.high(result)

+    result[stmtIndex] = body

+

+    # Parse through EOL again

+    inc(index, value.parse_thru_eol(index))

+

+

+proc parse_until_symbol(node: PNimrodNode, value: string, index: var int): bool {.compiletime.} =

+    ## Parses a string until a $ symbol is encountered, if

+    ## two $$'s are encountered in a row, a split will happen

+    ## removing one of the $'s from the resulting output

+    var splitValue: string

+    var read = value.parseUntil(splitValue, '$', index)

+    var insertionPoint = node.len

+

+    inc(index, read + 1)

+    if index < value.len:

+

+        case value[index]

+        of '$':

+            # Check for duplicate `$`, meaning this is an escaped $

+            node.add newCall("add", ident("result"), newStrLitNode("$"))

+            inc(index)

+

+        of '(':

+            # Check for open `(`, which means parse as simple single-line expression.

+            trim_eol(splitValue)

+            read = value.parse_to_close(index) + 1

+            node.add newCall("add", ident("result"),

+                newCall(bindSym"strip", parseExpr("$" & value.substring(index, read)))

+            )

+            inc(index, read)

+

+        of '{':

+            # Check for open `{`, which means open statement list

+            trim_eol(splitValue)

+            for s in value.parse_stmt_list(index):

+                node.add parseExpr(s)

+

+        else:

+            # Otherwise parse while valid `identChars` and make expression w/ $

+            var identifier: string

+            read = value.parseWhile(identifier, identChars, index)

+

+            if identifier in ["for", "while"]:

+                ## for/while means open simple statement

+                trim_eol(splitValue)

+                node.add value.parse_simple_statement(index)

+

+            elif identifier in ["if", "when", "case", "try"]:

+                ## if/when/case/try means complex statement

+                trim_eol(splitValue)

+                node.add value.parse_complex_stmt(identifier, index)

+

+            elif identifier.len > 0:

+                ## Treat as simple variable

+                node.add newCall("add", ident("result"), newCall("$", ident(identifier)))

+                inc(index, read)

+

+        result = true

+

+    # Insert

+    if splitValue.len > 0:

+        node.insert insertionPoint, newCall("add", ident("result"), newStrLitNode(splitValue))

+

+

+proc parse_template(node: PNimrodNode, value: string) =

+    ## Parses through entire template, outputing valid

+    ## Nimrod code into the input `node` AST.

+    var index = 0

+    while index < value.len and

+          parse_until_symbol(node, value, index): nil

+

+

+macro tmpli*(body: expr): stmt =

+    result = newStmtList()

+

+    result.add parseExpr("result = \"\"")

+

+    var value = if body.kind in nnkStrLit..nnkTripleStrLit: body.strVal

+                else: body[1].strVal

+

+    parse_template(result, reindent(value))

+

+

+macro tmpl*(body: expr): stmt =

+    result = newStmtList()

+

+    var value = if body.kind in nnkStrLit..nnkTripleStrLit: body.strVal

+                else: body[1].strVal

+

+    parse_template(result, reindent(value))

+

+

+# Run tests

+when isMainModule:

+    include otests
+    echo "Success"