diff options
Diffstat (limited to 'lib/impure/re.nim')
| -rw-r--r--[-rwxr-xr-x] | lib/impure/re.nim | 779 |
1 files changed, 514 insertions, 265 deletions
diff --git a/lib/impure/re.nim b/lib/impure/re.nim index bce786087..053c6ab55 100755..100644 --- a/lib/impure/re.nim +++ b/lib/impure/re.nim @@ -1,235 +1,542 @@ # # -# Nimrod's Runtime Library -# (c) Copyright 2010 Andreas Rumpf +# Nim's Runtime Library +# (c) Copyright 2012 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. # -## Regular expression support for Nimrod. Consider using the pegs module -## instead. +when defined(js): + {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE; See jsre for JS backend.".} + +## Regular expression support for Nim. +## ## This module is implemented by providing a wrapper around the -## `PRCE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ -## C library. This means that your application will depend on the PRCE +## `PCRE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_ +## C library. This means that your application will depend on the PCRE ## library's licence when using this module, which should not be a problem ## though. -## PRCE's licence follows: ## -## .. include:: ../doc/regexprs.txt +## .. note:: There are also alternative nimble packages such as [tinyre](https://github.com/khchen/tinyre) +## and [regex](https://github.com/nitely/nim-regex). +## +## PCRE's licence follows: +## +## .. include:: ../../doc/regexprs.txt ## +runnableExamples: + ## Unless specified otherwise, `start` parameter in each proc indicates + ## where the scan starts, but outputs are relative to the start of the input + ## string, not to `start`: + doAssert find("uxabc", re"(?<=x|y)ab", start = 1) == 2 # lookbehind assertion + doAssert find("uxabc", re"ab", start = 3) == -1 # we're past `start` => not found + doAssert not match("xabc", re"^abc$", start = 1) + # can't match start of string since we're starting at 1 + import - pcre, strutils + std/[pcre, strutils, rtarrays] + +when defined(nimPreviewSlimSystem): + import std/syncio const - MaxSubpatterns* = 10 + MaxSubpatterns* = 20 ## defines the maximum number of subpatterns that can be captured. - ## More subpatterns cannot be captured! + ## This limit still exists for `replacef` and `parallelReplace`. type - TRegExFlag* = enum ## options for regular expressions - reIgnoreCase = 0, ## do caseless matching - reMultiLine = 1, ## ``^`` and ``$`` match newlines within data - reDotAll = 2, ## ``.`` matches anything including NL - reExtended = 3 ## ignore whitespace and ``#`` comments - - TRegExDesc {.pure, final.} = object - h: PPcre - - TRegEx* = ref TRegExDesc ## a compiled regular expression - - EInvalidRegEx* = object of EInvalidValue + RegexFlag* = enum ## options for regular expressions + reIgnoreCase = 0, ## do caseless matching + reMultiLine = 1, ## `^` and `$` match newlines within data + reDotAll = 2, ## `.` matches anything including NL + reExtended = 3, ## ignore whitespace and `#` comments + reStudy = 4 ## study the expression (may be omitted if the + ## expression will be used only once) + + RegexDesc = object + h: ptr Pcre + e: ptr ExtraData + + Regex* = ref RegexDesc ## a compiled regular expression + + RegexError* = object of ValueError ## is raised if the pattern is no valid regular expression. -proc rawCompile(pattern: string, flags: cint): PPcre = +when defined(gcDestructors): + when defined(nimAllowNonVarDestructor): + proc `=destroy`(x: RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) + else: + proc `=destroy`(x: var RegexDesc) = + pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) + +proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} = + var e: ref RegexError + new(e) + e.msg = msg + raise e + +proc rawCompile(pattern: string, flags: cint): ptr Pcre = var - msg: CString - offset: cint - com = pcre.Compile(pattern, flags, addr(msg), addr(offset), nil) - if com == nil: - var e: ref EInvalidRegEx - new(e) - e.msg = $msg & "\n" & pattern & "\n" & repeatChar(offset) & "^\n" - raise e - return com - -proc finalizeRegEx(x: TRegEx) = + msg: cstring = "" + offset: cint = 0 + result = pcre.compile(pattern, flags, addr(msg), addr(offset), nil) + if result == nil: + raiseInvalidRegex($msg & "\n" & pattern & "\n" & spaces(offset) & "^\n") + +proc finalizeRegEx(x: Regex) = # XXX This is a hack, but PCRE does not export its "free" function properly. - # Sigh. The hack relies on PCRE's implementation (see ``pcre_get.c``). - # Fortunately the implementation is unlikely to change. + # Sigh. The hack relies on PCRE's implementation (see `pcre_get.c`). + # Fortunately the implementation is unlikely to change. pcre.free_substring(cast[cstring](x.h)) + if not isNil(x.e): + pcre.free_study(x.e) -proc re*(s: string, flags = {reExtended}): TRegEx = - ## Constructor of regular expressions. Note that Nimrod's - ## extended raw string literals supports this syntax ``re"[abc]"`` as - ## a short form for ``re(r"[abc]")``. - new(result, finalizeRegEx) - result.h = rawCompile(s, cast[cint](flags)) - -proc matchOrFind(s: string, pattern: TRegEx, matches: var openarray[string], - start, flags: cint): cint = +proc re*(s: string, flags = {reStudy}): Regex = + ## Constructor of regular expressions. + ## + ## Note that Nim's + ## extended raw string literals support the syntax `re"[abc]"` as + ## a short form for `re(r"[abc]")`. Also note that since this + ## compiles the regular expression, which is expensive, you should + ## avoid putting it directly in the arguments of the functions like + ## the examples show below if you plan to use it a lot of times, as + ## this will hurt performance immensely. (e.g. outside a loop, ...) + when defined(gcDestructors): + result = Regex() + else: + new(result, finalizeRegEx) + result.h = rawCompile(s, cast[cint](flags - {reStudy})) + if reStudy in flags: + var msg: cstring = "" + var options: cint = 0 + var hasJit: cint = 0 + if pcre.config(pcre.CONFIG_JIT, addr hasJit) == 0: + if hasJit == 1'i32: + options = pcre.STUDY_JIT_COMPILE + result.e = pcre.study(result.h, options, addr msg) + if not isNil(msg): raiseInvalidRegex($msg) + +proc rex*(s: string, flags = {reStudy, reExtended}): Regex = + ## Constructor for extended regular expressions. + ## + ## The extended means that comments starting with `#` and + ## whitespace are ignored. + result = re(s, flags) + +proc bufSubstr(b: cstring, sPos, ePos: int): string {.inline.} = + ## Return a Nim string built from a slice of a cstring buffer. + ## Don't assume cstring is '\0' terminated + let sz = ePos - sPos + result = newString(sz+1) + copyMem(addr(result[0]), unsafeAddr(b[sPos]), sz) + result.setLen(sz) + +proc matchOrFind(buf: cstring, pattern: Regex, matches: var openArray[string], + start, bufSize, flags: cint): cint = var - rawMatches: array[0..maxSubpatterns * 3 - 1, cint] - res = pcre.Exec(pattern.h, nil, s, len(s), start, flags, - cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3) + rtarray = initRtArray[cint]((matches.len+1)*3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize, start, flags, + cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return res for i in 1..int(res)-1: var a = rawMatches[i * 2] var b = rawMatches[i * 2 + 1] - if a >= 0'i32: matches[i-1] = copy(s, int(a), int(b)-1) + if a >= 0'i32: + matches[i-1] = bufSubstr(buf, int(a), int(b)) else: matches[i-1] = "" return rawMatches[1] - rawMatches[0] -proc matchOrFind(s: string, pattern: TRegEx, start, flags: cint): cint = - var rawMatches: array [0..maxSubpatterns * 3 - 1, cint] - result = pcre.Exec(pattern.h, nil, s, len(s), start, flags, - cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3) +const MaxReBufSize* = high(cint) + ## Maximum PCRE (API 1) buffer start/size equal to `high(cint)`, which even + ## for 64-bit systems can be either 2`31`:sup:-1 or 2`63`:sup:-1. + +proc findBounds*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), + ## and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `(-1,0)` is returned. + ## + ## Note: The memory for `matches` needs to be allocated before this function is + ## called, otherwise it will just remain empty. + var + rtarray = initRtArray[cint]((matches.len+1)*3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, + cast[ptr cint](rawMatches), (matches.len+1).cint*3) + if res < 0'i32: return (-1, 0) + for i in 1..int(res)-1: + var a = rawMatches[i * 2] + var b = rawMatches[i * 2 + 1] + if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b)) + else: matches[i-1] = "" + return (rawMatches[0].int, rawMatches[1].int - 1) + +proc findBounds*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): tuple[first, last: int] {.inline.} = + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing + ## is written into `matches` and `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[string](1) + let (first, last) = findBounds("Hello World", re"(W\w+)", matches) + doAssert first == 6 + doAssert last == 10 + doAssert matches[0] == "World" + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBounds*(buf: cstring, pattern: Regex, + matches: var openArray[tuple[first, last: int]], + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the starting position and end position of `pattern` in `buf` + ## (where `buf` has length `bufSize` and is not necessarily `'\0'` terminated), + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + var + rtarray = initRtArray[cint]((matches.len+1)*3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, + cast[ptr cint](rawMatches), (matches.len+1).cint*3) + if res < 0'i32: return (-1, 0) + for i in 1..int(res)-1: + var a = rawMatches[i * 2] + var b = rawMatches[i * 2 + 1] + if a >= 0'i32: matches[i-1] = (int(a), int(b)-1) + else: matches[i-1] = (-1,0) + return (rawMatches[0].int, rawMatches[1].int - 1) + +proc findBounds*(s: string, pattern: Regex, + matches: var openArray[tuple[first, last: int]], + start = 0): tuple[first, last: int] {.inline.} = + ## returns the starting position and end position of `pattern` in `s` + ## and the captured substrings in the array `matches`. + ## If it does not match, nothing is written into `matches` and + ## `(-1,0)` is returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + var matches = newSeq[tuple[first, last: int]](1) + let (first, last) = findBounds("Hello World", re"(\w+)", matches) + doAssert first == 0 + doAssert last == 4 + doAssert matches[0] == (0, 4) + result = findBounds(cstring(s), pattern, matches, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc findBoundsImpl(buf: cstring, pattern: Regex, + start = 0, bufSize = 0, flags = 0): tuple[first, last: int] = + var rtarray = initRtArray[cint](3) + let rawMatches = rtarray.getRawData + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags.int32, + cast[ptr cint](rawMatches), 3) + + if res < 0'i32: + result = (-1, 0) + else: + result = (int(rawMatches[0]), int(rawMatches[1]-1)) + +proc findBounds*(buf: cstring, pattern: Regex, + start = 0, bufSize: int): tuple[first, last: int] = + ## returns the `first` and `last` position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `(-1,0)` is returned. + var + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, + cast[ptr cint](rawMatches), 3) + if res < 0'i32: return (int(res), 0) + return (int(rawMatches[0]), int(rawMatches[1]-1)) + +proc findBounds*(s: string, pattern: Regex, + start = 0): tuple[first, last: int] {.inline.} = + ## returns the `first` and `last` position of `pattern` in `s`. + ## If it does not match, `(-1,0)` is returned. + ## + ## Note: there is a speed improvement if the matches do not need to be captured. + runnableExamples: + assert findBounds("01234abc89", re"abc") == (5,7) + result = findBounds(cstring(s), pattern, + min(start, MaxReBufSize), min(s.len, MaxReBufSize)) + +proc matchOrFind(buf: cstring, pattern: Regex, start, bufSize: int, flags: cint): cint = + var + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + result = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, flags, + cast[ptr cint](rawMatches), 3) if result >= 0'i32: result = rawMatches[1] - rawMatches[0] -proc match*(s: string, pattern: TRegEx, matches: var openarray[string], - start = 0): bool = - ## returns ``true`` if ``s[start..]`` matches the ``pattern`` and - ## the captured substrings in the array ``matches``. If it does not - ## match, nothing is written into ``matches`` and ``false`` is - ## returned. - return matchOrFind(s, pattern, matches, start, - pcre.ANCHORED) == cint(s.len - start) +proc matchLen*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length + ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = matchOrFind(cstring(s), pattern, matches, start.cint, s.len.cint, pcre.ANCHORED) -proc match*(s: string, pattern: TRegEx, start = 0): bool = - ## returns ``true`` if ``s[start..]`` matches the ``pattern``. - return matchOrFind(s, pattern, start, pcre.ANCHORED) == cint(s.len - start) +proc matchLen*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length + ## of zero can happen. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + return matchOrFind(buf, pattern, matches, start.cint, bufSize.cint, pcre.ANCHORED) + +proc matchLen*(s: string, pattern: Regex, start = 0): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length + ## of zero can happen. + ## + runnableExamples: + doAssert matchLen("abcdefg", re"cde", 2) == 3 + doAssert matchLen("abcdefg", re"abcde") == 5 + doAssert matchLen("abcdefg", re"cde") == -1 + result = matchOrFind(cstring(s), pattern, start.cint, s.len.cint, pcre.ANCHORED) -proc matchLen*(s: string, pattern: TRegEx, matches: var openarray[string], - start = 0): int = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length +proc matchLen*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int {.inline.} = + ## the same as `match`, but it returns the length of the match, + ## if there is no match, `-1` is returned. Note that a match length ## of zero can happen. - return matchOrFind(s, pattern, matches, start, pcre.ANCHORED) - -proc matchLen*(s: string, pattern: TRegEx, start = 0): int = - ## the same as ``match``, but it returns the length of the match, - ## if there is no match, -1 is returned. Note that a match length - ## of zero can happen. - return matchOrFind(s, pattern, start, pcre.ANCHORED) - -proc find*(s: string, pattern: TRegEx, matches: var openarray[string], - start = 0): int = - ## returns the starting position of ``pattern`` in ``s`` and the captured - ## substrings in the array ``matches``. If it does not match, nothing - ## is written into ``matches`` and -1 is returned. + result = matchOrFind(buf, pattern, start.cint, bufSize, pcre.ANCHORED) + +proc match*(s: string, pattern: Regex, start = 0): bool {.inline.} = + ## returns `true` if `s[start..]` matches the `pattern`. + result = matchLen(cstring(s), pattern, start, s.len) != -1 + +proc match*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): bool {.inline.} = + ## returns `true` if `s[start..]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is + ## returned. + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + runnableExamples: + import std/sequtils + var matches: array[2, string] + if match("abcdefg", re"c(d)ef(g)", matches, 2): + doAssert toSeq(matches) == @["d", "g"] + result = matchLen(cstring(s), pattern, matches, start, s.len) != -1 + +proc match*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): bool {.inline.} = + ## returns `true` if `buf[start..<bufSize]` matches the `pattern` and + ## the captured substrings in the array `matches`. If it does not + ## match, nothing is written into `matches` and `false` is + ## returned. + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = matchLen(buf, pattern, matches, start, bufSize) != -1 + +proc find*(buf: cstring, pattern: Regex, matches: var openArray[string], + start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. + ## `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. var - rawMatches: array[0..maxSubpatterns * 3 - 1, cint] - res = pcre.Exec(pattern.h, nil, s, len(s), start, 0'i32, - cast[ptr cint](addr(rawMatches)), maxSubpatterns * 3) + rtarray = initRtArray[cint]((matches.len+1)*3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, + cast[ptr cint](rawMatches), (matches.len+1).cint*3) if res < 0'i32: return res for i in 1..int(res)-1: var a = rawMatches[i * 2] var b = rawMatches[i * 2 + 1] - if a >= 0'i32: matches[i-1] = copy(s, int(a), int(b)-1) + if a >= 0'i32: matches[i-1] = bufSubstr(buf, int(a), int(b)) else: matches[i-1] = "" return rawMatches[0] -proc find*(s: string, pattern: TRegEx, start = 0): int = - ## returns the starting position of ``pattern`` in ``s``. If it does not - ## match, -1 is returned. - var matches: array[0..maxSubpatterns-1, string] - result = find(s, pattern, matches, start) - -iterator findAll*(s: string, pattern: TRegEx, start = 0): string = - ## yields all matching captures of pattern in `s`. - var matches: array[0..MaxSubpatterns-1, string] - var i = start - while true: - var j = find(s, pattern, matches, i) - if j < 0: break - i = j - for k in 0..maxSubPatterns-1: - if isNil(matches[k]): break - inc(i, matches[k].len) - yield matches[k] - -proc findAll*(s: string, pattern: TRegEx, start = 0): seq[string] = - ## returns all matching captures of pattern in `s`. - ## If it does not match, @[] is returned. - accumulateResult(findAll(s, pattern, start)) - -template `=~` *(s: string, pattern: TRegEx): expr = - ## This calls ``match`` with an implicit declared ``matches`` array that - ## can be used in the scope of the ``=~`` call: - ## - ## .. code-block:: nimrod +proc find*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): int {.inline.} = + ## returns the starting position of `pattern` in `s` and the captured + ## substrings in the array `matches`. If it does not match, nothing + ## is written into `matches` and `-1` is returned. ## - ## if line =~ re"\s*(\w+)\s*\=\s*(\w+)": - ## # matches a key=value pair: - ## echo("Key: ", matches[0]) - ## echo("Value: ", matches[1]) - ## elif line =~ re"\s*(\#.*)": - ## # matches a comment - ## # note that the implicit ``matches`` array is different from the - ## # ``matches`` array of the first branch - ## echo("comment: ", matches[0]) - ## else: - ## echo("syntax error") + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. + result = find(cstring(s), pattern, matches, start, s.len) + +proc find*(buf: cstring, pattern: Regex, start = 0, bufSize: int): int = + ## returns the starting position of `pattern` in `buf`, + ## where `buf` has length `bufSize` (not necessarily `'\0'` terminated). + ## If it does not match, `-1` is returned. + var + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, start.cint, 0'i32, + cast[ptr cint](rawMatches), 3) + if res < 0'i32: return res + return rawMatches[0] + +proc find*(s: string, pattern: Regex, start = 0): int {.inline.} = + ## returns the starting position of `pattern` in `s`. If it does not + ## match, `-1` is returned. We start the scan at `start`. + runnableExamples: + doAssert find("abcdefg", re"cde") == 2 + doAssert find("abcdefg", re"abc") == 0 + doAssert find("abcdefg", re"zz") == -1 # not found + doAssert find("abcdefg", re"cde", start = 2) == 2 # still 2 + doAssert find("abcdefg", re"cde", start = 3) == -1 # we're past the start position + doAssert find("xabc", re"(?<=x|y)abc", start = 1) == 1 + # lookbehind assertion `(?<=x|y)` can look behind `start` + result = find(cstring(s), pattern, start, s.len) + +iterator findAll*(s: string, pattern: Regex, start = 0): string = + ## Yields all matching *substrings* of `s` that match `pattern`. ## - when not definedInScope(matches): - var matches: array[0..maxSubPatterns-1, string] + ## Note that since this is an iterator you should not modify the string you + ## are iterating over: bad things could happen. + var + i = int32(start) + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + while true: + let res = pcre.exec(pattern.h, pattern.e, s, len(s).cint, i, 0'i32, + cast[ptr cint](rawMatches), 3) + if res < 0'i32: break + let a = rawMatches[0] + let b = rawMatches[1] + if a == b and a == i: break + yield substr(s, int(a), int(b)-1) + i = b + +iterator findAll*(buf: cstring, pattern: Regex, start = 0, bufSize: int): string = + ## Yields all matching `substrings` of `s` that match `pattern`. + ## + ## Note that since this is an iterator you should not modify the string you + ## are iterating over: bad things could happen. + var + i = int32(start) + rtarray = initRtArray[cint](3) + rawMatches = rtarray.getRawData + while true: + let res = pcre.exec(pattern.h, pattern.e, buf, bufSize.cint, i, 0'i32, + cast[ptr cint](rawMatches), 3) + if res < 0'i32: break + let a = rawMatches[0] + let b = rawMatches[1] + if a == b and a == i: break + var str = newString(b-a) + copyMem(str[0].addr, unsafeAddr(buf[a]), b-a) + yield str + i = b + +proc findAll*(s: string, pattern: Regex, start = 0): seq[string] {.inline.} = + ## returns all matching `substrings` of `s` that match `pattern`. + ## If it does not match, `@[]` is returned. + result = @[] + for x in findAll(s, pattern, start): result.add x + +template `=~` *(s: string, pattern: Regex): untyped = + ## This calls `match` with an implicit declared `matches` array that + ## can be used in the scope of the `=~` call: + runnableExamples: + proc parse(line: string): string = + if line =~ re"\s*(\w+)\s*\=\s*(\w+)": # matches a key=value pair: + result = $(matches[0], matches[1]) + elif line =~ re"\s*(\#.*)": # matches a comment + # note that the implicit `matches` array is different from 1st branch + result = $(matches[0],) + else: raiseAssert "unreachable" + doAssert not declared(matches) + doAssert parse("NAME = LENA") == """("NAME", "LENA")""" + doAssert parse(" # comment ... ") == """("# comment ... ",)""" + bind MaxSubpatterns + when not declaredInScope(matches): + var matches {.inject.}: array[MaxSubpatterns, string] match(s, pattern, matches) # ------------------------- more string handling ------------------------------ -proc contains*(s: string, pattern: TRegEx, start = 0): bool = - ## same as ``find(s, pattern, start) >= 0`` +proc contains*(s: string, pattern: Regex, start = 0): bool {.inline.} = + ## same as `find(s, pattern, start) >= 0` return find(s, pattern, start) >= 0 -proc contains*(s: string, pattern: TRegEx, matches: var openArray[string], - start = 0): bool = - ## same as ``find(s, pattern, matches, start) >= 0`` +proc contains*(s: string, pattern: Regex, matches: var openArray[string], + start = 0): bool {.inline.} = + ## same as `find(s, pattern, matches, start) >= 0` + ## + ## .. note:: The memory for `matches` needs to be allocated before this function is called, otherwise it will just remain empty. return find(s, pattern, matches, start) >= 0 -proc startsWith*(s: string, prefix: TRegEx): bool = +proc startsWith*(s: string, prefix: Regex): bool {.inline.} = ## returns true if `s` starts with the pattern `prefix` result = matchLen(s, prefix) >= 0 -proc endsWith*(s: string, suffix: TRegEx): bool = - ## returns true if `s` ends with the pattern `prefix` +proc endsWith*(s: string, suffix: Regex): bool {.inline.} = + ## returns true if `s` ends with the pattern `suffix` for i in 0 .. s.len-1: if matchLen(s, suffix, i) == s.len - i: return true -proc replace*(s: string, sub: TRegEx, by: string): string = +proc replace*(s: string, sub: Regex, by = ""): string = + ## Replaces `sub` in `s` by the string `by`. Captures cannot be + ## accessed in `by`. + runnableExamples: + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)") == "; " + doAssert "var1=key; var2=key2".replace(re"(\w+)=(\w+)", "?") == "?; ?" + result = "" + var prev = 0 + var flags = int32(0) + while prev < s.len: + var match = findBoundsImpl(s.cstring, sub, prev, s.len, flags) + flags = 0 + if match.first < 0: break + add(result, substr(s, prev, match.first-1)) + add(result, by) + if match.first > match.last: + # 0-len match + flags = pcre.NOTEMPTY_ATSTART + prev = match.last + 1 + add(result, substr(s, prev)) + +proc replacef*(s: string, sub: Regex, by: string): string = ## Replaces `sub` in `s` by the string `by`. Captures can be accessed in `by` - ## with the notation ``$i`` and ``$#`` (see strutils.`%`). Examples: - ## - ## .. code-block:: nimrod - ## "var1=key; var2=key2".replace(re"(\w+)'='(\w+)", "$1<-$2$2") - ## - ## Results in: - ## - ## .. code-block:: nimrod - ## - ## "var1<-keykey; val2<-key2key2" + ## with the notation `$i` and `$#` (see strutils.\`%\`). + runnableExamples: + doAssert "var1=key; var2=key2".replacef(re"(\w+)=(\w+)", "$1<-$2$2") == + "var1<-keykey; var2<-key2key2" result = "" - var i = 0 - var caps: array[0..maxSubpatterns-1, string] - while i < s.len: - var x = matchLen(s, sub, caps, i) - if x <= 0: - add(result, s[i]) - inc(i) - else: - addf(result, by, caps) - inc(i, x) - # copy the rest: - add(result, copy(s, i)) - -proc parallelReplace*(s: string, subs: openArray[ - tuple[pattern: TRegEx, repl: string]]): string = + var caps: array[MaxSubpatterns, string] + var prev = 0 + while prev < s.len: + var match = findBounds(s, sub, caps, prev) + if match.first < 0: break + add(result, substr(s, prev, match.first-1)) + addf(result, by, caps) + if match.last + 1 == prev: break + prev = match.last + 1 + add(result, substr(s, prev)) + +proc multiReplace*(s: string, subs: openArray[ + tuple[pattern: Regex, repl: string]]): string = ## Returns a modified copy of `s` with the substitutions in `subs` ## applied in parallel. result = "" var i = 0 - var caps: array[0..maxSubpatterns-1, string] + var caps: array[MaxSubpatterns, string] while i < s.len: block searchSubs: for j in 0..high(subs): @@ -241,122 +548,64 @@ proc parallelReplace*(s: string, subs: openArray[ add(result, s[i]) inc(i) # copy the rest: - add(result, copy(s, i)) - + add(result, substr(s, i)) + proc transformFile*(infile, outfile: string, - subs: openArray[tuple[pattern: TRegEx, repl: string]]) = + subs: openArray[tuple[pattern: Regex, repl: string]]) = ## reads in the file `infile`, performs a parallel replacement (calls - ## `parallelReplace`) and writes back to `outfile`. Calls ``quit`` if an + ## `parallelReplace`) and writes back to `outfile`. Raises `IOError` if an ## error occurs. This is supposed to be used for quick scripting. var x = readFile(infile) - if not isNil(x): - var f: TFile - if open(f, outfile, fmWrite): - write(f, x.parallelReplace(subs)) - close(f) - else: - quit("cannot open for writing: " & outfile) - else: - quit("cannot open for reading: " & infile) - -iterator split*(s: string, sep: TRegEx): string = + writeFile(outfile, x.multiReplace(subs)) + +iterator split*(s: string, sep: Regex; maxsplit = -1): string = ## Splits the string `s` into substrings. ## - ## Substrings are separated by the regular expression `sep`. - ## Examples: - ## - ## .. code-block:: nimrod - ## for word in split("00232this02939is39an22example111", re"\d+"): - ## writeln(stdout, word) - ## - ## Results in: - ## - ## .. code-block:: nimrod - ## "this" - ## "is" - ## "an" - ## "example" - ## - var - first = 0 - last = 0 - while last < len(s): - var x = matchLen(s, sep, last) - if x > 0: inc(last, x) - first = last - while last < len(s): + ## Substrings are separated by the regular expression `sep` + ## (and the portion matched by `sep` is not returned). + runnableExamples: + import std/sequtils + doAssert toSeq(split("00232this02939is39an22example111", re"\d+")) == + @["", "this", "is", "an", "example", ""] + var last = 0 + var splits = maxsplit + var x = -1 + if len(s) == 0: + last = 1 + if matchLen(s, sep, 0) == 0: + x = 0 + while last <= len(s): + var first = last + var sepLen = 1 + if x == 0: inc(last) + while last < len(s): x = matchLen(s, sep, last) - if x > 0: break - if first < last: - yield copy(s, first, last-1) + if x >= 0: + sepLen = x + break + inc(last) + if splits == 0: last = len(s) + yield substr(s, first, last-1) + if splits == 0: break + dec(splits) + inc(last, sepLen) -proc split*(s: string, sep: TRegEx): seq[string] = - ## Splits the string `s` into substrings. - accumulateResult(split(s, sep)) - -proc escapeRe*(s: string): string = - ## escapes `s` so that it is matched verbatim when used as a regular +proc split*(s: string, sep: Regex, maxsplit = -1): seq[string] {.inline.} = + ## Splits the string `s` into a seq of substrings. + ## + ## The portion matched by `sep` is not returned. + result = @[] + for x in split(s, sep, maxsplit): result.add x + +proc escapeRe*(s: string): string = + ## escapes `s` so that it is matched verbatim when used as a regular ## expression. result = "" for c in items(s): case c of 'a'..'z', 'A'..'Z', '0'..'9', '_': result.add(c) - else: + else: result.add("\\x") result.add(toHex(ord(c), 2)) - -const ## common regular expressions - reIdentifier* = r"\b[a-zA-Z_]+[a-zA-Z_0-9]*\b" ## describes an identifier - reNatural* = r"\b\d+\b" ## describes a natural number - reInteger* = r"\b[-+]?\d+\b" ## describes an integer - reHex* = r"\b0[xX][0-9a-fA-F]+\b" ## describes a hexadecimal number - reBinary* = r"\b0[bB][01]+\b" ## describes a binary number (example: 0b11101) - reOctal* = r"\b0[oO][0-7]+\b" ## describes an octal number (example: 0o777) - reFloat* = r"\b[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?\b" - ## describes a floating point number - reEmail* = r"\b[a-zA-Z0-9!#$%&'*+/=?^_`{|}~\-]+(?:\. &" & - r"[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)" & - r"*@(?:[a-zA-Z0-9](?:[a-zA-Z0-9-]*[a-zA-Z0-9])?\.)+" & - r"(?:[a-zA-Z]{2}|com|org|" & - r"net|gov|mil|biz|info|mobi|name|aero|jobs|museum)\b" - ## describes a common email address - reURL* = r"\b(http(s)?|ftp|gopher|telnet|file|notes|ms\-help):" & - r"((//)|(\\\\))+[\w\d:#@%/;$()~_?\+\-\=\\\.\&]*\b" - ## describes an URL - -when isMainModule: - assert match("(a b c)", re"\( .* \)") - assert match("WHiLe", re("while", {reIgnoreCase})) - - assert "0158787".match(re"\d+") - assert "ABC 0232".match(re"\w+\s+\d+") - assert "ABC".match(re"\d+ | \w+") - - assert matchLen("key", re(reIdentifier)) == 3 - - var pattern = re"[a-z0-9]+\s*=\s*[a-z0-9]+" - assert matchLen("key1= cal9", pattern) == 11 - - assert find("_____abc_______", re"abc") == 5 - - var matches: array[0..5, string] - if match("abcdefg", re"c(d)ef(g)", matches, 2): - assert matches[0] == "d" - assert matches[1] == "g" - else: - assert false - - if "abc" =~ re"(a)bcxyz|(\w+)": - assert matches[1] == "abc" - else: - assert false - - assert "var1=key; var2=key2".endsWith(re"\w+=\w+") - assert("var1=key; var2=key2".replace(re"(\w+)=(\w+)", "$1<-$2$2") == - "var1<-keykey; var2<-key2key2") - - for word in split("00232this02939is39an22example111", re"\d+"): - writeln(stdout, word) - |