diff options
| -rw-r--r-- | koch.nim | 8 | ||||
| -rw-r--r-- | tools/atlas/atlas.md | 77 | ||||
| -rw-r--r-- | tools/atlas/atlas.nim | 390 | ||||
| -rw-r--r-- | tools/atlas/osutils.nim | 59 | ||||
| -rw-r--r-- | tools/atlas/packagesjson.nim | 117 | ||||
| -rw-r--r-- | tools/atlas/parse_requires.nim | 101 |
6 files changed, 751 insertions, 1 deletions
diff --git a/koch.nim b/koch.nim index 4ba0d0eb5..da7576e65 100644 --- a/koch.nim +++ b/koch.nim @@ -222,7 +222,13 @@ proc buildTools(args: string = "") = # `-d:nimDebugUtils` only makes sense when temporarily editing/debugging compiler # `-d:debug` should be changed to a flag that doesn't require re-compiling nim # `--opt:speed` is a sensible default even for a debug build, it doesn't affect nim stacktraces - nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, outputName = "nim_dbg") + nimCompileFold("Compile nim_dbg", "compiler/nim.nim", options = + "--opt:speed --stacktrace -d:debug --stacktraceMsgs -d:nimCompilerStacktraceHints " & args, + outputName = "nim_dbg") + + nimCompileFold("Compile atlas", "tools/atlas/atlas.nim", options = "-d:release " & args, + outputName = "atlas") + proc nsis(latest: bool; args: string) = bundleNimbleExe(latest, args) diff --git a/tools/atlas/atlas.md b/tools/atlas/atlas.md new file mode 100644 index 000000000..a36817dc5 --- /dev/null +++ b/tools/atlas/atlas.md @@ -0,0 +1,77 @@ +# Atlas Package Cloner + +Atlas is a simple package cloner tool that automates some of the +workflows and needs for Nim's stdlib evolution. + +Atlas is compatible with Nimble in the sense that it supports the Nimble +file format. + + +## How it works + +Atlas uses git commits internally; version requirements are translated +to git commits via `git show-ref --tags`. + +Atlas uses URLs internally; Nimble package names are translated to URLs +via Nimble's `packages.json` file. + +Atlas does not call the Nim compiler for a build, instead it creates/patches +a `nim.cfg` file for the compiler. For example: + +``` +############# begin Atlas config section ########## +--noNimblePath +--path:"../nimx" +--path:"../sdl2/src" +--path:"../opengl/src" +############# end Atlas config section ########## +``` + +The version selection is deterministic, it picks up the *minimum* required +version. Thanks to this design, lock files are not required. + + +## Dependencies + +Dependencies are neither installed globally, nor locally into the current +project. Instead a "workspace" is used. The workspace is the nearest parent +directory of the current directory that does not contain a `.git` subdirectory. +Dependencies are managed as **siblings**, not as children. Dependencies are +kept as git repositories. + +Thanks to this setup, it's easy to develop multiple projects at the same time. + +A project plus its dependencies are stored in a workspace: + + $workspace / main project + $workspace / dependency A + $workspace / dependency B + + +No attempts are being made at keeping directory hygiene inside the +workspace, you're supposed to create appropriate `$workspace` directories +at your own leisure. + + +## Commands + +Atlas supports the following commands: + + +### Clone <url> + +Clones a URL and all of its dependencies (recursively) into the workspace. +Creates or patches a `nim.cfg` file with the required `--path` entries. + + +### Clone <package name> + +The `<package name>` is translated into an URL via `packages.json` and +then `clone <url>` is performed. + + +### Search <term term2 term3 ...> + +Search the package index `packages.json` for a package that the given terms +in its description (or name or list of tags). + diff --git a/tools/atlas/atlas.nim b/tools/atlas/atlas.nim new file mode 100644 index 000000000..bc1cc6ae3 --- /dev/null +++ b/tools/atlas/atlas.nim @@ -0,0 +1,390 @@ +# +# Atlas Package Cloner +# (c) Copyright 2021 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Simple tool to automate frequent workflows: Can "clone" +## a Nimble dependency and its dependencies recursively. + +import std/[parseopt, strutils, os, osproc, sequtils, unicode, tables, sets] +import parse_requires, osutils, packagesjson + +const + Version = "0.1" + Usage = "atlas - Nim Package Manager Version " & Version & """ + + (c) 2021 Andreas Rumpf +Usage: + atlas [options] [command] [arguments] +Command: + clone url|pkgname clone a package and all of its dependencies + search keyw keywB... search for package that contains the given keywords + +Options: + --keepCommits do not perform any `git checkouts` + --version show the version + --help show this help +""" + +proc writeHelp() = + stdout.write(Usage) + stdout.flushFile() + quit(0) + +proc writeVersion() = + stdout.write(Version & "\n") + stdout.flushFile() + quit(0) + +type + PackageName = distinct string + DepRelation = enum + normal, strictlyLess, strictlyGreater + + Dependency = object + name: PackageName + url, commit: string + rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too. + AtlasContext = object + projectDir, workspace: string + hasPackageList: bool + keepCommits: bool + p: Table[string, string] # name -> url mapping + processed: HashSet[string] # the key is (url / commit) + errors: int + +const + InvalidCommit = "<invalid commit>" + +proc toDepRelation(s: string): DepRelation = + case s + of "<": strictlyLess + of ">": strictlyGreater + else: normal + +proc isCleanGit(dir: string): string = + result = "" + let (outp, status) = osproc.execCmdEx("git diff") + if outp.len != 0: + result = "'git diff' not empty" + elif status != 0: + result = "'git diff' returned non-zero" + +proc message(c: var AtlasContext; category: string; p: PackageName; args: varargs[string]) = + var msg = category & "(" & p.string & ")" + for a in args: + msg.add ' ' + msg.add a + stdout.writeLine msg + inc c.errors + +proc warn(c: var AtlasContext; p: PackageName; args: varargs[string]) = + message(c, "[Warning] ", p, args) + +proc error(c: var AtlasContext; p: PackageName; args: varargs[string]) = + message(c, "[Error] ", p, args) + +proc sameVersionAs(tag, ver: string): bool = + const VersionChars = {'0'..'9', '.'} + + proc safeCharAt(s: string; i: int): char {.inline.} = + if i >= 0 and i < s.len: s[i] else: '\0' + + let idx = find(tag, ver) + if idx >= 0: + # we found the version as a substring inside the `tag`. But we + # need to watch out the the boundaries are not part of a + # larger/different version number: + result = safeCharAt(tag, idx-1) notin VersionChars and + safeCharAt(tag, idx+ver.len) notin VersionChars + +proc versionToCommit(d: Dependency): string = + let (outp, status) = osproc.execCmdEx("git show-ref --tags") + if status == 0: + var useNextOne = false + for line in splitLines(outp): + let commitsAndTags = strutils.splitWhitespace(line) + if commitsAndTags.len == 2: + case d.rel + of normal: + if commitsAndTags[1].sameVersionAs(d.commit): + return commitsAndTags[0] + of strictlyLess: + if d.commit == InvalidCommit or not commitsAndTags[1].sameVersionAs(d.commit): + return commitsAndTags[0] + of strictlyGreater: + if commitsAndTags[1].sameVersionAs(d.commit): + useNextOne = true + elif useNextOne: + return commitsAndTags[0] + + return "" + +proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) = + let (outp, status) = osproc.execCmdEx("git checkout " & quoteShell(commit)) + if status != 0: + error(c, p, "could not checkout commit", commit) + +proc gitPull(c: var AtlasContext; p: PackageName) = + let (_, status) = osproc.execCmdEx("git pull") + if status != 0: + error(c, p, "could not 'git pull'") + +proc updatePackages(c: var AtlasContext) = + if dirExists(c.workspace / PackagesDir): + withDir(c.workspace / PackagesDir): + gitPull(c, PackageName PackagesDir) + else: + withDir c.workspace: + let err = cloneUrl("https://github.com/nim-lang/packages", PackagesDir, false) + if err != "": + error c, PackageName(PackagesDir), err + +proc fillPackageLookupTable(c: var AtlasContext) = + if not c.hasPackageList: + c.hasPackageList = true + updatePackages(c) + let plist = getPackages(c.workspace) + for entry in plist: + c.p[unicode.toLower entry.name] = entry.url + +proc toUrl(c: var AtlasContext; p: string): string = + if p.isUrl: + result = p + else: + fillPackageLookupTable(c) + result = c.p.getOrDefault(unicode.toLower p) + if result.len == 0: + inc c.errors + +proc toName(p: string): PackageName = + if p.isUrl: + result = PackageName splitFile(p).name + else: + result = PackageName p + +proc needsCommitLookup(commit: string): bool {.inline} = + '.' in commit or commit == InvalidCommit + +proc checkoutCommit(c: var AtlasContext; w: Dependency) = + let dir = c.workspace / w.name.string + withDir dir: + if w.commit.len == 0 or cmpIgnoreCase(w.commit, "#head") == 0: + gitPull(c, w.name) + else: + let err = isCleanGit(dir) + if err != "": + warn c, w.name, err + else: + let requiredCommit = if needsCommitLookup(w.commit): versionToCommit(w) else: w.commit + let (cc, status) = osproc.execCmdEx("git log -n 1 --format=%H") + let currentCommit = strutils.strip(cc) + if requiredCommit == "" or status != 0: + if requiredCommit == "" and w.commit == InvalidCommit: + warn c, w.name, "package has no tagged releases" + else: + warn c, w.name, "cannot find specified version/commit", w.commit + else: + if currentCommit != requiredCommit: + # checkout the later commit: + # git merge-base --is-ancestor <commit> <commit> + let (mergeBase, status) = osproc.execCmdEx("git merge-base " & + currentCommit.quoteShell & " " & requiredCommit.quoteShell) + if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit): + # conflict resolution: pick the later commit: + if mergeBase == currentCommit: + checkoutGitCommit(c, w.name, requiredCommit) + else: + checkoutGitCommit(c, w.name, requiredCommit) + when false: + warn c, w.name, "do not know which commit is more recent:", + currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)" + +proc findNimbleFile(c: AtlasContext; dep: Dependency): string = + result = c.workspace / dep.name.string / (dep.name.string & ".nimble") + if not fileExists(result): + result = "" + for x in walkFiles(c.workspace / dep.name.string / "*.nimble"): + if result.len == 0: + result = x + else: + # ambiguous .nimble file + return "" + +proc addUniqueDep(c: var AtlasContext; work: var seq[Dependency]; + tokens: seq[string]) = + let oldErrors = c.errors + let url = toUrl(c, tokens[0]) + if oldErrors != c.errors: + warn c, toName(tokens[0]), "cannot resolve package name" + elif not c.processed.containsOrIncl(url / tokens[2]): + work.add Dependency(name: toName(tokens[0]), url: url, commit: tokens[2], + rel: toDepRelation(tokens[1])) + +proc collectNewDeps(c: var AtlasContext; work: var seq[Dependency]; + dep: Dependency; result: var seq[string]; + isMainProject: bool) = + let nimbleFile = findNimbleFile(c, dep) + if nimbleFile != "": + let nimbleInfo = extractRequiresInfo(nimbleFile) + for r in nimbleInfo.requires: + var tokens: seq[string] = @[] + for token in tokenizeRequires(r): + tokens.add token + if tokens.len == 1: + # nimx uses dependencies like 'requires "sdl2"'. + # Via this hack we map them to the first tagged release. + # (See the `isStrictlySmallerThan` logic.) + tokens.add "<" + tokens.add InvalidCommit + elif tokens.len == 2 and tokens[1].startsWith("#"): + # Dependencies can also look like 'requires "sdl2#head" + var commit = tokens[1] + tokens[1] = "==" + tokens.add commit + + if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0: + c.addUniqueDep work, tokens + + result.add dep.name.string / nimbleInfo.srcDir + else: + result.add dep.name.string + +proc clone(c: var AtlasContext; start: string): seq[string] = + # non-recursive clone. + let oldErrors = c.errors + var work = @[Dependency(name: toName(start), url: toUrl(c, start), commit: "")] + + if oldErrors != c.errors: + error c, toName(start), "cannot resolve package name" + return + + c.projectDir = work[0].name.string + result = @[] + var i = 0 + while i < work.len: + let w = work[i] + let oldErrors = c.errors + if not dirExists(c.workspace / w.name.string): + withDir c.workspace: + let err = cloneUrl(w.url, w.name.string, false) + if err != "": + error c, w.name, err + if oldErrors == c.errors: + if not c.keepCommits: checkoutCommit(c, w) + # even if the checkout fails, we can make use of the somewhat + # outdated .nimble file to clone more of the most likely still relevant + # dependencies: + collectNewDeps(c, work, w, result, i == 0) + inc i + +const + configPatternBegin = "############# begin Atlas config section ##########\n" + configPatternEnd = "############# end Atlas config section ##########\n" + +proc patchNimCfg(c: AtlasContext; deps: seq[string]) = + var paths = "--noNimblePath\n" + for d in deps: + paths.add "--path:\"../" & d.replace("\\", "/") & "\"\n" + + let cfg = c.projectDir / "nim.cfg" + var cfgContent = configPatternBegin & paths & configPatternEnd + if not fileExists(cfg): + writeFile(cfg, cfgContent) + else: + let content = readFile(cfg) + let start = content.find(configPatternBegin) + if start >= 0: + cfgContent = content.substr(0, start-1) & cfgContent + let theEnd = content.find(configPatternEnd, start) + if theEnd >= 0: + cfgContent.add content.substr(theEnd+len(configPatternEnd)) + else: + cfgContent = content & "\n" & cfgContent + if cfgContent != content: + # do not touch the file if nothing changed + # (preserves the file date information): + writeFile(cfg, cfgContent) + +proc error*(msg: string) = + when defined(debug): + writeStackTrace() + quit "[Error] " & msg + +proc main = + var action = "" + var args: seq[string] = @[] + template singleArg() = + if args.len != 1: + error action & " command takes a single package name" + + template noArgs() = + if args.len != 0: + error action & " command takes no arguments" + + var c = AtlasContext( + projectDir: getCurrentDir(), + workspace: getCurrentDir()) + + for kind, key, val in getopt(): + case kind + of cmdArgument: + if action.len == 0: + action = key.normalize + else: + args.add key + of cmdLongOption, cmdShortOption: + case normalize(key) + of "help", "h": writeHelp() + of "version", "v": writeVersion() + of "keepcommits": c.keepCommits = true + else: writeHelp() + of cmdEnd: assert false, "cannot happen" + + while c.workspace.len > 0 and dirExists(c.workspace / ".git"): + c.workspace = c.workspace.parentDir() + + case action + of "": + error "No action." + of "clone": + singleArg() + let deps = clone(c, args[0]) + patchNimCfg c, deps + if c.errors > 0: + error "There were problems." + of "refresh": + noArgs() + updatePackages(c) + of "search", "list": + updatePackages(c) + search getPackages(c.workspace), args + else: + error "Invalid action: " & action + +when isMainModule: + main() + +when false: + # some testing code for the `patchNimCfg` logic: + var c = AtlasContext( + projectDir: getCurrentDir(), + workspace: getCurrentDir().parentDir) + + patchNimCfg(c, @[PackageName"abc", PackageName"xyz"]) + +when false: + assert sameVersionAs("v0.2.0", "0.2.0") + assert sameVersionAs("v1", "1") + + assert sameVersionAs("1.90", "1.90") + + assert sameVersionAs("v1.2.3-zuzu", "1.2.3") + assert sameVersionAs("foo-1.2.3.4", "1.2.3.4") + + assert not sameVersionAs("foo-1.2.3.4", "1.2.3") + assert not sameVersionAs("foo", "1.2.3") + assert not sameVersionAs("", "1.2.3") diff --git a/tools/atlas/osutils.nim b/tools/atlas/osutils.nim new file mode 100644 index 000000000..31392b113 --- /dev/null +++ b/tools/atlas/osutils.nim @@ -0,0 +1,59 @@ +## OS utilities like 'withDir'. +## (c) 2021 Andreas Rumpf + +import os, strutils, osproc + +template withDir*(dir, body) = + let oldDir = getCurrentDir() + try: + setCurrentDir(dir) + body + finally: + setCurrentDir(oldDir) + +proc isUrl*(x: string): bool = + x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://") + +proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string = + ## Returns an error message on error or else "". + result = "" + var modUrl = + if url.startsWith("git://") and cloneUsingHttps: + "https://" & url[6 .. ^1] + else: url + + # github + https + trailing url slash causes a + # checkout/ls-remote to fail with Repository not found + var isGithub = false + if modUrl.contains("github.com") and modUrl.endswith("/"): + modUrl = modUrl[0 .. ^2] + isGithub = true + + let (_, exitCode) = execCmdEx("git ls-remote --quiet --tags " & modUrl) + var xcode = exitCode + if isGithub and exitCode != QuitSuccess: + # retry multiple times to avoid annoying github timeouts: + for i in 0..4: + os.sleep(4000) + xcode = execCmdEx("git ls-remote --quiet --tags " & modUrl)[1] + if xcode == QuitSuccess: break + + if xcode == QuitSuccess: + # retry multiple times to avoid annoying github timeouts: + let cmd = "git clone " & modUrl & " " & dest + for i in 0..4: + if execShellCmd(cmd) == 0: return "" + os.sleep(4000) + result = "exernal program failed: " & cmd + elif not isGithub: + let (_, exitCode) = execCmdEx("hg identify " & modUrl) + if exitCode == QuitSuccess: + let cmd = "hg clone " & modUrl & " " & dest + for i in 0..4: + if execShellCmd(cmd) == 0: return "" + os.sleep(4000) + result = "exernal program failed: " & cmd + else: + result = "Unable to identify url: " & modUrl + else: + result = "Unable to identify url: " & modUrl diff --git a/tools/atlas/packagesjson.nim b/tools/atlas/packagesjson.nim new file mode 100644 index 000000000..e1e23ee29 --- /dev/null +++ b/tools/atlas/packagesjson.nim @@ -0,0 +1,117 @@ + +import std / [json, os, sets, strutils] +import osutils + +type + Package* = ref object + # Required fields in a package. + name*: string + url*: string # Download location. + license*: string + downloadMethod*: string + description*: string + tags*: seq[string] # \ + # From here on, optional fields set to the empty string if not available. + version*: string + dvcsTag*: string + web*: string # Info url for humans. + +proc optionalField(obj: JsonNode, name: string, default = ""): string = + if hasKey(obj, name) and obj[name].kind == JString: + result = obj[name].str + else: + result = default + +proc requiredField(obj: JsonNode, name: string): string = + result = optionalField(obj, name, "") + +proc fromJson*(obj: JSonNode): Package = + result = Package() + result.name = obj.requiredField("name") + if result.name.len == 0: return nil + result.version = obj.optionalField("version") + result.url = obj.requiredField("url") + if result.url.len == 0: return nil + result.downloadMethod = obj.requiredField("method") + if result.downloadMethod.len == 0: return nil + result.dvcsTag = obj.optionalField("dvcs-tag") + result.license = obj.optionalField("license") + result.tags = @[] + for t in obj["tags"]: + result.tags.add(t.str) + result.description = obj.requiredField("description") + result.web = obj.optionalField("web") + +const PackagesDir* = "packages" + +proc getPackages*(workspaceDir: string): seq[Package] = + result = @[] + var uniqueNames = initHashSet[string]() + var jsonFiles = 0 + for kind, path in walkDir(workspaceDir / PackagesDir): + if kind == pcFile and path.endsWith(".json"): + inc jsonFiles + let packages = json.parseFile(path) + for p in packages: + let pkg = p.fromJson() + if pkg != nil and not uniqueNames.containsOrIncl(pkg.name): + result.add(pkg) + +proc `$`*(pkg: Package): string = + result = pkg.name & ":\n" + result &= " url: " & pkg.url & " (" & pkg.downloadMethod & ")\n" + result &= " tags: " & pkg.tags.join(", ") & "\n" + result &= " description: " & pkg.description & "\n" + result &= " license: " & pkg.license & "\n" + if pkg.web.len > 0: + result &= " website: " & pkg.web & "\n" + +proc search*(pkgList: seq[Package]; terms: seq[string]) = + var found = false + template onFound = + echo pkg + echo("") + found = true + break forPackage + + for pkg in pkgList: + if terms.len > 0: + block forPackage: + for term in terms: + let word = term.toLower + # Search by name. + if word in pkg.name.toLower: + onFound() + # Search by tag. + for tag in pkg.tags: + if word in tag.toLower: + onFound() + else: + echo(pkg) + echo(" ") + + if not found and terms.len > 0: + echo("No package found.") + +type PkgCandidates* = array[3, seq[Package]] + +proc determineCandidates*(pkgList: seq[Package]; + terms: seq[string]): PkgCandidates = + result[0] = @[] + result[1] = @[] + result[2] = @[] + for pkg in pkgList: + block termLoop: + for term in terms: + let word = term.toLower + if word == pkg.name.toLower: + result[0].add pkg + break termLoop + elif word in pkg.name.toLower: + result[1].add pkg + break termLoop + else: + for tag in pkg.tags: + if word in tag.toLower: + result[2].add pkg + break termLoop diff --git a/tools/atlas/parse_requires.nim b/tools/atlas/parse_requires.nim new file mode 100644 index 000000000..7e26a1656 --- /dev/null +++ b/tools/atlas/parse_requires.nim @@ -0,0 +1,101 @@ +## Utility API for Nim package managers. +## (c) 2021 Andreas Rumpf + +import std / strutils +import ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils] + +type + NimbleFileInfo* = object + requires*: seq[string] + srcDir*: string + tasks*: seq[(string, string)] + +proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) = + case n.kind + of nkStmtList, nkStmtListExpr: + for child in n: + extract(child, conf, result) + of nkCallKinds: + if n[0].kind == nkIdent: + case n[0].ident.s + of "requires": + for i in 1..<n.len: + var ch = n[i] + while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0: ch = ch.lastSon + if ch.kind in {nkStrLit..nkTripleStrLit}: + result.requires.add ch.strVal + else: + localError(conf, ch.info, "'requires' takes string literals") + of "task": + if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}: + result.tasks.add((n[1].ident.s, n[2].strVal)) + else: discard + of nkAsgn, nkFastAsgn: + if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0: + if n[1].kind in {nkStrLit..nkTripleStrLit}: + result.srcDir = n[1].strVal + else: + localError(conf, n[1].info, "assignments to 'srcDir' must be string literals") + else: + discard + +proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo = + ## Extract the `requires` information from a Nimble file. This does **not** + ## evaluate the Nimble file. Errors are produced on stderr/stdout and are + ## formatted as the Nim compiler does it. The parser uses the Nim compiler + ## as an API. The result can be empty, this is not an error, only parsing + ## errors are reported. + var conf = newConfigRef() + conf.foreignPackageNotes = {} + conf.notes = {} + conf.mainPackageNotes = {} + + let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile) + var parser: Parser + if setupParser(parser, fileIdx, newIdentCache(), conf): + extract(parseAll(parser), conf, result) + closeParser(parser) + +const Operators* = {'<', '>', '=', '&', '@', '!', '^'} + +proc token(s: string; idx: int; lit: var string): int = + var i = idx + if i >= s.len: return i + while s[i] in Whitespace: inc(i) + case s[i] + of Letters, '#': + lit.add s[i] + inc i + while i < s.len and s[i] notin (Whitespace + {'@', '#'}): + lit.add s[i] + inc i + of '0'..'9': + while i < s.len and s[i] in {'0'..'9', '.'}: + lit.add s[i] + inc i + of '"': + inc i + while i < s.len and s[i] != '"': + lit.add s[i] + inc i + inc i + of Operators: + while i < s.len and s[i] in Operators: + lit.add s[i] + inc i + else: + lit.add s[i] + inc i + result = i + +iterator tokenizeRequires*(s: string): string = + var start = 0 + var tok = "" + while start < s.len: + tok.setLen 0 + start = token(s, start, tok) + yield tok + +when isMainModule: + for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"): + echo x |