diff options
| -rw-r--r-- | lib/pure/parsesql.nim | 185 | ||||
| -rw-r--r-- | tests/stdlib/tparsesql.nim | 407 |
2 files changed, 196 insertions, 396 deletions
diff --git a/lib/pure/parsesql.nim b/lib/pure/parsesql.nim index f266beef7..ae192ab9a 100644 --- a/lib/pure/parsesql.nim +++ b/lib/pure/parsesql.nim @@ -55,6 +55,13 @@ const ";", ":", ",", "(", ")", "[", "]", "." ] + reservedKeywords = @[ + # statements + "select", "from", "where", "group", "limit", "having", + # functions + "count", + ] + proc open(L: var SqlLexer, input: Stream, filename: string) = lexbase.open(L, input) L.filename = filename @@ -274,16 +281,16 @@ proc getSymbol(c: var SqlLexer, tok: var Token) = c.bufpos = pos tok.kind = tkIdentifier -proc getQuotedIdentifier(c: var SqlLexer, tok: var Token) = +proc getQuotedIdentifier(c: var SqlLexer, tok: var Token, quote='\"') = var pos = c.bufpos + 1 var buf = c.buf tok.kind = tkQuotedIdentifier while true: var ch = buf[pos] - if ch == '\"': - if buf[pos+1] == '\"': + if ch == quote: + if buf[pos+1] == quote: inc(pos, 2) - add(tok.literal, '\"') + add(tok.literal, quote) else: inc(pos) break @@ -442,7 +449,8 @@ proc getTok(c: var SqlLexer, tok: var Token) = add(tok.literal, '.') of '0'..'9': getNumeric(c, tok) of '\'': getString(c, tok, tkStringConstant) - of '"': getQuotedIdentifier(c, tok) + of '"': getQuotedIdentifier(c, tok, '"') + of '`': getQuotedIdentifier(c, tok, '`') of lexbase.EndOfFile: tok.kind = tkEof tok.literal = "[EOF]" @@ -450,7 +458,7 @@ proc getTok(c: var SqlLexer, tok: var Token) = '\128'..'\255': getSymbol(c, tok) of '+', '-', '*', '/', '<', '>', '=', '~', '!', '@', '#', '%', - '^', '&', '|', '`', '?': + '^', '&', '|', '?': getOperator(c, tok) else: add(tok.literal, c.buf[c.bufpos]) @@ -504,6 +512,7 @@ type nkPrefix, nkInfix, nkCall, + nkPrGroup, nkColumnReference, nkReferences, nkDefault, @@ -700,7 +709,8 @@ proc identOrLiteral(p: var SqlParser): SqlNode = getTok(p) of tkParLe: getTok(p) - result = parseExpr(p) + result = newNode(nkPrGroup) + result.add(parseExpr(p)) eat(p, tkParRi) else: sqlError(p, "expression expected") @@ -752,7 +762,7 @@ proc lowestExprAux(p: var SqlParser, v: var SqlNode, limit: int): int = result = opPred while opPred > limit: node = newNode(nkInfix) - opNode = newNode(nkIdent, p.tok.literal) + opNode = newNode(nkIdent, p.tok.literal.toLower()) getTok(p) result = lowestExprAux(p, v2, opPred) node.add(opNode) @@ -1112,7 +1122,8 @@ proc parseSelect(p: var SqlParser): SqlNode = join.add(newNode(nkIdent, "")) getTok(p) else: - join.add(parseExpr(p)) + join.add(newNode(nkIdent, p.tok.literal.toLower())) + getTok(p) eat(p, "join") join.add(parseFromItem(p)) eat(p, "on") @@ -1167,8 +1178,6 @@ proc parse(p: var SqlParser): SqlNode = if p.tok.kind == tkEof: break eat(p, tkSemicolon) - if result.len == 1: - result = result.sons[0] proc close(p: var SqlParser) = ## closes the parser `p`. The associated input stream is closed too. @@ -1198,44 +1207,25 @@ type upperCase: bool buffer: string -proc add(s: var SqlWriter, thing: string) = +proc add(s: var SqlWriter, thing: char) = s.buffer.add(thing) -proc add(s: var SqlWriter, thing: char) = +proc add(s: var SqlWriter, thing: string) = + if s.buffer.len > 0 and s.buffer[^1] notin {' ', '\L', '(', '.'}: + s.buffer.add(" ") s.buffer.add(thing) proc addKeyw(s: var SqlWriter, thing: string) = - if s.buffer.len > 0 and s.buffer[^1] notin " ,\L(": - s.buffer.add(" ") + var keyw = thing if s.upperCase: - s.buffer.add(thing.toUpper()) - else: - s.buffer.add(thing) - s.buffer.add(" ") - -proc rm(s: var SqlWriter, chars = " \L,") = - while s.buffer.len > 0 and s.buffer[^1] in chars: - s.buffer = s.buffer[0..^2] - -proc newLine(s: var SqlWriter) = - s.rm(" \L") - s.buffer.add("\L") - for i in 0..<s.indent: - s.buffer.add(" ") - -template inner(s: var SqlWriter, body: untyped) = - inc s.indent - s.newLine() - body - dec s.indent - -template innerKeyw(s: var SqlWriter, keyw: string, body: untyped) = - s.newLine() - s.addKeyw(keyw) - inc s.indent - s.newLine() - body - dec s.indent + keyw = keyw.toUpper() + s.add(keyw) + +proc addIden(s: var SqlWriter, thing: string) = + var iden = thing + if iden.toLower() in reservedKeywords: + iden = '"' & iden & '"' + s.add(iden) proc ra(n: SqlNode, s: var SqlWriter) @@ -1247,17 +1237,30 @@ proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") = ra(n.sons[i], s) s.add(suffix) +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',') = + if n.len > 0: + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + +proc addMulti(s: var SqlWriter, n: SqlNode, sep = ',', prefix, suffix: char) = + if n.len > 0: + s.add(prefix) + for i in 0 .. n.len-1: + if i > 0: s.add(sep) + ra(n.sons[i], s) + s.add(suffix) + proc ra(n: SqlNode, s: var SqlWriter) = if n == nil: return case n.kind of nkNone: discard of nkIdent: - if allCharsInSet(n.strVal, {'\33'..'\127'}): + if allCharsInSet(n.strVal, {'\33'..'\127'}) and n.strVal.toLower() notin reservedKeywords: s.add(n.strVal) else: s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"") of nkStringLit: - # TODO add e'' as an option? s.add(escape(n.strVal, "'", "'")) of nkBitStringLit: s.add("b'" & n.strVal & "'") @@ -1277,33 +1280,33 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("null") of nkDot: ra(n.sons[0], s) - s.add(".") + s.add('.') ra(n.sons[1], s) of nkDotDot: ra(n.sons[0], s) s.add(". .") ra(n.sons[1], s) of nkPrefix: - s.add('(') ra(n.sons[0], s) s.add(' ') ra(n.sons[1], s) - s.add(')') of nkInfix: - s.add('(') ra(n.sons[1], s) s.add(' ') ra(n.sons[0], s) s.add(' ') ra(n.sons[2], s) - s.add(')') of nkCall, nkColumnReference: ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(", ") + if i > 1: s.add(',') ra(n.sons[i], s) s.add(')') + of nkPrGroup: + s.add('(') + s.addMulti(n) + s.add(')') of nkReferences: s.addKeyw("references") ra(n.sons[0], s) @@ -1324,55 +1327,43 @@ proc ra(n: SqlNode, s: var SqlWriter) = of nkIdentity: s.addKeyw("identity") of nkColumnDef: - s.add("\n ") rs(n, s, "", "", " ") of nkStmtList: for i in 0..n.len-1: ra(n.sons[i], s) - s.add("\n") + s.add(';') of nkInsert: assert n.len == 3 s.addKeyw("insert into") ra(n.sons[0], s) - s.add(" ") + s.add(' ') ra(n.sons[1], s) if n.sons[2].kind == nkDefault: s.addKeyw("default values") else: - s.newLine() ra(n.sons[2], s) - s.rm(" ") - s.add(';') of nkUpdate: - s.innerKeyw("update"): - ra(n.sons[0], s) - s.innerKeyw("set"): - var L = n.len - for i in 1 .. L-2: - if i > 1: s.add(", ") - var it = n.sons[i] - assert it.kind == nkAsgn - ra(it, s) + s.addKeyw("update") + ra(n.sons[0], s) + s.addKeyw("set") + var L = n.len + for i in 1 .. L-2: + if i > 1: s.add(", ") + var it = n.sons[i] + assert it.kind == nkAsgn + ra(it, s) ra(n.sons[L-1], s) - s.add(';') of nkDelete: s.addKeyw("delete from") ra(n.sons[0], s) ra(n.sons[1], s) - s.add(';') of nkSelect, nkSelectDistinct: s.addKeyw("select") if n.kind == nkSelectDistinct: s.addKeyw("distinct") - s.inner: - for son in n.sons[0].sons: - ra(son, s) - s.add(',') - s.newLine() - s.rm() + s.addMulti(n.sons[0]) for i in 1 .. n.len-1: ra(n.sons[i], s) - s.add(';') of nkSelectColumns: assert(false) of nkSelectPair: @@ -1385,12 +1376,9 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) else: assert n.sons[0].kind == nkSelect - s.add("(") - s.inner: - ra(n.sons[0], s) - s.rm("; \L") - s.newLine() - s.add(")") + s.add('(') + ra(n.sons[0], s) + s.add(')') if n.sons.len == 2: s.addKeyw("as") ra(n.sons[1], s) @@ -1399,30 +1387,30 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.add(" = ") ra(n.sons[1], s) of nkFrom: - s.innerKeyw("from"): - rs(n, s, "", "", ", ") + s.addKeyw("from") + s.addMulti(n) of nkGroup: - s.innerKeyw("group by"): - rs(n, s, "", "", ", ") + s.addKeyw("group by") + s.addMulti(n) of nkLimit: - s.innerKeyw("limit"): - rs(n, s, "", "", ", ") + s.addKeyw("limit") + s.addMulti(n) of nkHaving: - s.innerKeyw("having"): - rs(n, s, "", "", ", ") + s.addKeyw("having") + s.addMulti(n) of nkOrder: - s.innerKeyw("order by"): - rs(n, s, "", "", ", ") + s.addKeyw("order by") + s.addMulti(n) of nkJoin: var joinType = n.sons[0].strVal if joinType == "": joinType = "join" else: joinType &= " " & "join" - s.innerKeyw(joinType): - ra(n.sons[1], s) - s.innerKeyw("on"): - ra(n.sons[2], s) + s.addKeyw(joinType) + ra(n.sons[1], s) + s.addKeyw("on") + ra(n.sons[2], s) of nkDesc: ra(n.sons[0], s) s.addKeyw("desc") @@ -1438,10 +1426,8 @@ proc ra(n: SqlNode, s: var SqlWriter) = s.addKeyw("values") rs(n, s) of nkWhere: - s.newLine() s.addKeyw("where") - s.inner: - ra(n.sons[0], s) + ra(n.sons[0], s) of nkCreateTable, nkCreateTableIfNotExists: s.addKeyw("create table") if n.kind == nkCreateTableIfNotExists: @@ -1449,7 +1435,7 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) s.add('(') for i in 1..n.len-1: - if i > 1: s.add(",") + if i > 1: s.add(',') ra(n.sons[i], s) s.add(");") of nkCreateType, nkCreateTypeIfNotExists: @@ -1459,7 +1445,6 @@ proc ra(n: SqlNode, s: var SqlWriter) = ra(n.sons[0], s) s.addKeyw("as") ra(n.sons[1], s) - s.add(';') of nkCreateIndex, nkCreateIndexIfNotExists: s.addKeyw("create index") if n.kind == nkCreateIndexIfNotExists: diff --git a/tests/stdlib/tparsesql.nim b/tests/stdlib/tparsesql.nim index fe64d3416..3dc949ea1 100644 --- a/tests/stdlib/tparsesql.nim +++ b/tests/stdlib/tparsesql.nim @@ -1,346 +1,149 @@ discard """ file: "tparsesql.nim" - output: '''select - foo -from - table; -select - foo -from - table; -select - foo -from - table -limit - 10; -select - foo, - bar, - baz -from - table -limit - 10; -select - foo as bar -from - table; -select - foo as foo_prime, - bar as bar_prime, - baz as baz_prime -from - table; -select - * -from - table; -select - * -from - table -where - ((a = b) and (c = d)); -select - * -from - table -where - (not b); -select - * -from - table -where - (a and (not b)); -select - * -from - table -where - (((a = b) and (c = d)) or ((n is null) and (((not b) + 1) = 3))); -select - * -from - table -having - ((a = b) and (c = d)); -select - a, - b -from - table -group by - a; -select - a, - b -from - table -group by - 1, 2; -select - t.a -from - t as t; -select - a, - b -from - ( - select - * - from - t - ); -select - a, - b -from - ( - select - * - from - t - ) as foo; -select - a, - b -from - ( - select - * - from - ( - select - * - from - ( - select - * - from - ( - select - * - from - inner as inner1 - ) as inner2 - ) as inner3 - ) as inner4 - ) as inner5; -select - a, - b -from - ( - select - * - from - a - ), ( - select - * - from - b - ), ( - select - * - from - c - ); -select - * -from - Products -where - (Price BETWEEN (10 AND 20)); -select - id -from - a -join - b -on - (a.id == b.id); -select - id -from - a -join - ( - select - id - from - c - ) as b -on - (a.id == b.id); -select - id -from - a -INNER join - b -on - (a.id == b.id); -select - id -from - a -OUTER join - b -on - (a.id == b.id); -select - id -from - a -CROSS join - b -on - (a.id == b.id); -create type happiness as enum ('happy', 'very happy', 'ecstatic'); -create table holidays( - num_weeks int, - happiness happiness); -create index table1_attr1 on table1(attr1); -select - * -from - myTab -where - (col1 = 'happy'); - -insert into Customers (CustomerName, ContactName, Address, City, PostalCode, Country) -values ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); -insert into TableName default values; - -update - Customers -set - ContactName = 'Alfred Schmidt', City = 'Frankfurt' -where - (CustomerID = 1); -delete from table_name; -delete from table_name; -select - * -from - Customers; -select - * -from - Customers -where - ((((CustomerName LIKE 'L%') OR (CustomerName LIKE 'R%')) OR (CustomerName LIKE 'W%')) AND (Country = 'USA')) -order by - CustomerName; - -''' """ import parsesql -echo $parseSQL "SELECT foo FROM table;" -echo $parseSQL "SELECT foo FROM table" -echo $parseSQL "SELECT foo FROM table limit 10" -echo $parseSQL "SELECT foo, bar, baz FROM table limit 10" -echo $parseSQL "SELECT foo AS bar FROM table" -echo $parseSQL "SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table" -echo $parseSQL "SELECT * FROM table" +doAssert $parseSQL("SELECT foo FROM table;") == "select foo from table;" +doAssert $parseSQL(""" +SELECT + CustomerName, + ContactName, + Address, + City, + PostalCode, + Country, + CustomerName, + ContactName, + Address, + City, + PostalCode, + Country +FROM table;""") == "select CustomerName, ContactName, Address, City, PostalCode, Country, CustomerName, ContactName, Address, City, PostalCode, Country from table;" + +doAssert $parseSQL("SELECT foo FROM table limit 10") == "select foo from table limit 10;" +doAssert $parseSQL("SELECT foo, bar, baz FROM table limit 10") == "select foo, bar, baz from table limit 10;" +doAssert $parseSQL("SELECT foo AS bar FROM table") == "select foo as bar from table;" +doAssert $parseSQL("SELECT foo AS foo_prime, bar AS bar_prime, baz AS baz_prime FROM table") == "select foo as foo_prime, bar as bar_prime, baz as baz_prime from table;" +doAssert $parseSQL("SELECT * FROM table") == "select * from table;" + + #TODO add count(*) -#echo $parseSQL "SELECT COUNT(*) FROM table" -echo $parseSQL """ +#doAssert $parseSQL("SELECT COUNT(*) FROM table" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a = b and c = d -""" -echo $parseSQL """ +""") == "select * from table where a = b and c = d;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE not b -""" -echo $parseSQL """ +""") == "select * from table where not b;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a and not b -""" -echo $parseSQL """ +""") == "select * from table where a and not b;" + +doAssert $parseSQL(""" SELECT * FROM table WHERE a = b and c = d or n is null and not b + 1 = 3 -""" -echo $parseSQL """ +""") == "select * from table where a = b and c = d or n is null and not b + 1 = 3;" + +doAssert $parseSQL(""" +SELECT * FROM table +WHERE (a = b and c = d) or (n is null and not b + 1 = 3) +""") == "select * from table where(a = b and c = d) or (n is null and not b + 1 = 3);" + +doAssert $parseSQL(""" SELECT * FROM table HAVING a = b and c = d -""" -echo $parseSQL """ +""") == "select * from table having a = b and c = d;" + +doAssert $parseSQL(""" SELECT a, b FROM table GROUP BY a -""" -echo $parseSQL """ +""") == "select a, b from table group by a;" + +doAssert $parseSQL(""" SELECT a, b FROM table GROUP BY 1, 2 -""" -echo $parseSQL "SELECT t.a FROM t as t" -echo $parseSQL """ +""") == "select a, b from table group by 1, 2;" + +doAssert $parseSQL("SELECT t.a FROM t as t") == "select t.a from t as t;" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM t ) -""" -echo $parseSQL """ +""") == "select a, b from(select * from t);" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM t ) as foo -""" -echo $parseSQL """ +""") == "select a, b from(select * from t) as foo;" + +doAssert $parseSQL(""" SELECT a, b FROM ( SELECT * FROM ( SELECT * FROM ( SELECT * FROM ( - SELECT * FROM inner as inner1 + SELECT * FROM innerTable as inner1 ) as inner2 ) as inner3 ) as inner4 ) as inner5 -""" -echo $parseSQL """ +""") == "select a, b from(select * from(select * from(select * from(select * from innerTable as inner1) as inner2) as inner3) as inner4) as inner5;" + +doAssert $parseSQL(""" SELECT a, b FROM (SELECT * FROM a), (SELECT * FROM b), (SELECT * FROM c) -""" -echo $parseSQL """ +""") == "select a, b from(select * from a),(select * from b),(select * from c);" + +doAssert $parseSQL(""" SELECT * FROM Products WHERE Price BETWEEN 10 AND 20; -""" -echo $parseSQL """ +""") == "select * from Products where Price between 10 and 20;" + +doAssert $parseSQL(""" SELECT id FROM a JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a JOIN (SELECT id from c) as b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a join(select id from c) as b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a INNER JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a inner join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a OUTER JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a outer join b on a.id == b.id;" + +doAssert $parseSQL(""" SELECT id FROM a CROSS JOIN b ON a.id == b.id -""" -echo $parseSQL """ +""") == "select id from a cross join b on a.id == b.id;" + +doAssert $parseSQL(""" CREATE TYPE happiness AS ENUM ('happy', 'very happy', 'ecstatic'); CREATE TABLE holidays ( num_weeks int, @@ -348,29 +151,41 @@ CREATE TABLE holidays ( ); CREATE INDEX table1_attr1 ON table1(attr1); SELECT * FROM myTab WHERE col1 = 'happy'; -""" -echo $parseSQL """ +""") == "create type happiness as enum ('happy' , 'very happy' , 'ecstatic' ); create table holidays(num_weeks int , happiness happiness );; create index table1_attr1 on table1(attr1 );; select * from myTab where col1 = 'happy';" + +doAssert $parseSQL(""" INSERT INTO Customers (CustomerName, ContactName, Address, City, PostalCode, Country) VALUES ('Cardinal', 'Tom B. Erichsen', 'Skagen 21', 'Stavanger', '4006', 'Norway'); -""" -echo $parseSQL """ +""") == "insert into Customers (CustomerName , ContactName , Address , City , PostalCode , Country ) values ('Cardinal' , 'Tom B. Erichsen' , 'Skagen 21' , 'Stavanger' , '4006' , 'Norway' );" + +doAssert $parseSQL(""" INSERT INTO TableName DEFAULT VALUES -""" -echo $parseSQL """ +""") == "insert into TableName default values;" + +doAssert $parseSQL(""" UPDATE Customers SET ContactName = 'Alfred Schmidt', City= 'Frankfurt' WHERE CustomerID = 1; -""" -echo $parseSQL "DELETE FROM table_name;" -echo $parseSQL "DELETE * FROM table_name;" -echo $parseSQL """ +""") == "update Customers set ContactName = 'Alfred Schmidt' , City = 'Frankfurt' where CustomerID = 1;" + +doAssert $parseSQL("DELETE FROM table_name;") == "delete from table_name;" + +doAssert $parseSQL("DELETE * FROM table_name;") == "delete from table_name;" + +doAssert $parseSQL(""" --Select all: SELECT * FROM Customers; -""" -echo $parseSQL """ +""") == "select * from Customers;" + +doAssert $parseSQL(""" SELECT * FROM Customers WHERE (CustomerName LIKE 'L%' OR CustomerName LIKE 'R%' /*OR CustomerName LIKE 'S%' OR CustomerName LIKE 'T%'*/ OR CustomerName LIKE 'W%') AND Country='USA' ORDER BY CustomerName; -""" +""") == "select * from Customers where(CustomerName like 'L%' or CustomerName like 'R%' or CustomerName like 'W%') and Country = 'USA' order by CustomerName;" + +# parse keywords as identifires +doAssert $parseSQL(""" +SELECT `SELECT`, `FROM` as `GROUP` FROM `WHERE`; +""") == """select "SELECT", "FROM" as "GROUP" from "WHERE";""" |