summary refs log tree commit diff stats
path: root/atlas
diff options
context:
space:
mode:
Diffstat (limited to 'atlas')
-rw-r--r--atlas/atlas.nim1166
-rw-r--r--atlas/atlas.nim.cfg1
-rw-r--r--atlas/compiledpatterns.nim246
-rw-r--r--atlas/osutils.nim51
-rw-r--r--atlas/packagesjson.nim161
-rw-r--r--atlas/parse_requires.nim101
-rw-r--r--atlas/testdata.nim63
-rw-r--r--atlas/tests/balls.nimble32
-rw-r--r--atlas/tests/grok.nimble5
-rw-r--r--atlas/tests/nim-bytes2human.nimble7
-rw-r--r--atlas/tests/nim.cfg10
-rw-r--r--atlas/tests/npeg.nimble48
-rw-r--r--atlas/tests/packages/packages.json36
-rw-r--r--atlas/tests/sync.nimble10
-rw-r--r--atlas/tests/testes.nimble23
-rw-r--r--atlas/tests/ups.nimble13
-rw-r--r--atlas/tests/ws_conflict/atlas.workspace2
-rw-r--r--atlas/tests/ws_conflict/source/apkg/apkg.nimble4
-rw-r--r--atlas/tests/ws_conflict/source/bpkg@1.0/bpkg.nimble1
-rw-r--r--atlas/tests/ws_conflict/source/cpkg@1.0/cpkg.nimble1
-rw-r--r--atlas/tests/ws_conflict/source/cpkg@2.0/cpkg.nimble1
-rw-r--r--atlas/tests/ws_conflict/source/dpkg/dpkg.nimble0
-rw-r--r--atlas/tests/ws_conflict/url.rules1
23 files changed, 1983 insertions, 0 deletions
diff --git a/atlas/atlas.nim b/atlas/atlas.nim
new file mode 100644
index 000000000..9793f2637
--- /dev/null
+++ b/atlas/atlas.nim
@@ -0,0 +1,1166 @@
+#
+#           Atlas Package Cloner
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Simple tool to automate frequent workflows: Can "clone"
+## a Nimble dependency and its dependencies recursively.
+
+import std / [parseopt, strutils, os, osproc, tables, sets, json, jsonutils,
+  parsecfg, streams, terminal, strscans, hashes]
+import parse_requires, osutils, packagesjson, compiledpatterns
+
+from unicode import nil
+
+const
+  Version = "0.4"
+  LockFileName = "atlas.lock"
+  AtlasWorkspace = "atlas.workspace"
+  Usage = "atlas - Nim Package Cloner Version " & Version & """
+
+  (c) 2021 Andreas Rumpf
+Usage:
+  atlas [options] [command] [arguments]
+Command:
+  init                  initializes the current directory as a workspace
+    --deps=DIR          use DIR as the directory for dependencies
+                        (default: store directly in the workspace)
+
+  use url|pkgname       clone a package and all of its dependencies and make
+                        it importable for the current project
+  clone url|pkgname     clone a package and all of its dependencies
+  update url|pkgname    update a package and all of its dependencies
+  install proj.nimble   use the .nimble file to setup the project's dependencies
+  search keyw keywB...  search for package that contains the given keywords
+  extract file.nimble   extract the requirements and custom commands from
+                        the given Nimble file
+  updateProjects [filter]
+                        update every project that has a remote
+                        URL that matches `filter` if a filter is given
+  updateDeps [filter]
+                        update every dependency that has a remote
+                        URL that matches `filter` if a filter is given
+  tag [major|minor|patch]
+                        add and push a new tag, input must be one of:
+                        ['major'|'minor'|'patch'] or a SemVer tag like ['1.0.3']
+                        or a letter ['a'..'z']: a.b.c.d.e.f.g
+  build|test|doc|tasks  currently delegates to `nimble build|test|doc`
+  task <taskname>       currently delegates to `nimble <taskname>`
+  env <nimversion>      setup a Nim virtual environment
+
+Options:
+  --keepCommits         do not perform any `git checkouts`
+  --cfgHere             also create/maintain a nim.cfg in the current
+                        working directory
+  --workspace=DIR       use DIR as workspace
+  --project=DIR         use DIR as the current project
+  --genlock             generate a lock file (use with `clone` and `update`)
+  --uselock             use the lock file for the build
+  --autoinit            auto initialize a workspace
+  --colors=on|off       turn on|off colored output
+  --showGraph           show the dependency graph
+  --version             show the version
+  --help                show this help
+"""
+
+proc writeHelp() =
+  stdout.write(Usage)
+  stdout.flushFile()
+  quit(0)
+
+proc writeVersion() =
+  stdout.write(Version & "\n")
+  stdout.flushFile()
+  quit(0)
+
+const
+  MockupRun = defined(atlasTests)
+  TestsDir = "atlas/tests"
+
+type
+  LockOption = enum
+    noLock, genLock, useLock
+
+  LockFileEntry = object
+    dir, url, commit: string
+
+  PackageName = distinct string
+  CfgPath = distinct string # put into a config `--path:"../x"`
+  DepRelation = enum
+    normal, strictlyLess, strictlyGreater
+
+  SemVerField = enum
+    major, minor, patch
+
+  Dependency = object
+    name: PackageName
+    url, commit: string
+    rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
+    self: int # position in the graph
+    parents: seq[int] # why we need this dependency
+    active: bool
+  DepGraph = object
+    nodes: seq[Dependency]
+    processed: Table[string, int] # the key is (url / commit)
+    byName: Table[PackageName, seq[int]]
+
+  AtlasContext = object
+    projectDir, workspace, depsDir, currentDir: string
+    hasPackageList: bool
+    keepCommits: bool
+    cfgHere: bool
+    usesOverrides: bool
+    p: Table[string, string] # name -> url mapping
+    errors, warnings: int
+    overrides: Patterns
+    lockOption: LockOption
+    lockFileToWrite: seq[LockFileEntry]
+    lockFileToUse: Table[string, LockFileEntry]
+    when MockupRun:
+      step: int
+      mockupSuccess: bool
+    noColors: bool
+    showGraph: bool
+
+proc `==`*(a, b: CfgPath): bool {.borrow.}
+
+proc `==`*(a, b: PackageName): bool {.borrow.}
+proc hash*(a: PackageName): Hash {.borrow.}
+
+const
+  InvalidCommit = "<invalid commit>"
+  ProduceTest = false
+
+type
+  Command = enum
+    GitDiff = "git diff",
+    GitTag = "git tag",
+    GitTags = "git show-ref --tags",
+    GitLastTaggedRef = "git rev-list --tags --max-count=1",
+    GitDescribe = "git describe",
+    GitRevParse = "git rev-parse",
+    GitCheckout = "git checkout",
+    GitPush = "git push origin",
+    GitPull = "git pull",
+    GitCurrentCommit = "git log -n 1 --format=%H"
+    GitMergeBase = "git merge-base"
+
+include testdata
+
+proc silentExec(cmd: string; args: openArray[string]): (string, int) =
+  var cmdLine = cmd
+  for i in 0..<args.len:
+    cmdLine.add ' '
+    cmdLine.add quoteShell(args[i])
+  result = osproc.execCmdEx(cmdLine)
+
+proc nimbleExec(cmd: string; args: openArray[string]) =
+  var cmdLine = "nimble " & cmd
+  for i in 0..<args.len:
+    cmdLine.add ' '
+    cmdLine.add quoteShell(args[i])
+  discard os.execShellCmd(cmdLine)
+
+proc exec(c: var AtlasContext; cmd: Command; args: openArray[string]): (string, int) =
+  when MockupRun:
+    assert TestLog[c.step].cmd == cmd, $(TestLog[c.step].cmd, cmd)
+    case cmd
+    of GitDiff, GitTag, GitTags, GitLastTaggedRef, GitDescribe, GitRevParse, GitPush, GitPull, GitCurrentCommit:
+      result = (TestLog[c.step].output, TestLog[c.step].exitCode)
+    of GitCheckout:
+      assert args[0] == TestLog[c.step].output
+    of GitMergeBase:
+      let tmp = TestLog[c.step].output.splitLines()
+      assert tmp.len == 4, $tmp.len
+      assert tmp[0] == args[0]
+      assert tmp[1] == args[1]
+      assert tmp[3] == ""
+      result[0] = tmp[2]
+      result[1] = TestLog[c.step].exitCode
+    inc c.step
+  else:
+    result = silentExec($cmd, args)
+    when ProduceTest:
+      echo "cmd ", cmd, " args ", args, " --> ", result
+
+proc cloneUrl(c: var AtlasContext; url, dest: string; cloneUsingHttps: bool): string =
+  when MockupRun:
+    result = ""
+  else:
+    result = osutils.cloneUrl(url, dest, cloneUsingHttps)
+    when ProduceTest:
+      echo "cloned ", url, " into ", dest
+
+template withDir*(c: var AtlasContext; dir: string; body: untyped) =
+  when MockupRun:
+    body
+  else:
+    let oldDir = getCurrentDir()
+    try:
+      when ProduceTest:
+        echo "Current directory is now ", dir
+      setCurrentDir(dir)
+      body
+    finally:
+      setCurrentDir(oldDir)
+
+proc extractRequiresInfo(c: var AtlasContext; nimbleFile: string): NimbleFileInfo =
+  result = extractRequiresInfo(nimbleFile)
+  when ProduceTest:
+    echo "nimble ", nimbleFile, " info ", result
+
+proc toDepRelation(s: string): DepRelation =
+  case s
+  of "<": strictlyLess
+  of ">": strictlyGreater
+  else: normal
+
+proc isCleanGit(c: var AtlasContext): string =
+  result = ""
+  let (outp, status) = exec(c, GitDiff, [])
+  if outp.len != 0:
+    result = "'git diff' not empty"
+  elif status != 0:
+    result = "'git diff' returned non-zero"
+
+proc message(c: var AtlasContext; category: string; p: PackageName; arg: string) =
+  var msg = category & "(" & p.string & ") " & arg
+  stdout.writeLine msg
+
+proc warn(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Warning] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgYellow, styleBright, "[Warning] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+  inc c.warnings
+
+proc error(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Error] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgRed, styleBright, "[Error] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+  inc c.errors
+
+proc info(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Info] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgGreen, styleBright, "[Info] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+
+template projectFromCurrentDir(): PackageName = PackageName(c.currentDir.splitPath.tail)
+
+proc readableFile(s: string): string = relativePath(s, getCurrentDir())
+
+proc sameVersionAs(tag, ver: string): bool =
+  const VersionChars = {'0'..'9', '.'}
+
+  proc safeCharAt(s: string; i: int): char {.inline.} =
+    if i >= 0 and i < s.len: s[i] else: '\0'
+
+  let idx = find(tag, ver)
+  if idx >= 0:
+    # we found the version as a substring inside the `tag`. But we
+    # need to watch out the the boundaries are not part of a
+    # larger/different version number:
+    result = safeCharAt(tag, idx-1) notin VersionChars and
+      safeCharAt(tag, idx+ver.len) notin VersionChars
+
+proc gitDescribeRefTag(c: var AtlasContext; commit: string): string =
+  let (lt, status) = exec(c, GitDescribe, ["--tags", commit])
+  result = if status == 0: strutils.strip(lt) else: ""
+
+proc getLastTaggedCommit(c: var AtlasContext): string =
+  let (ltr, status) = exec(c, GitLastTaggedRef, [])
+  if status == 0:
+    let lastTaggedRef = ltr.strip()
+    let lastTag = gitDescribeRefTag(c, lastTaggedRef)
+    if lastTag.len != 0:
+      result = lastTag
+
+proc versionToCommit(c: var AtlasContext; d: Dependency): string =
+  let (outp, status) = exec(c, GitTags, [])
+  if status == 0:
+    var useNextOne = false
+    for line in splitLines(outp):
+      let commitsAndTags = strutils.splitWhitespace(line)
+      if commitsAndTags.len == 2:
+        case d.rel
+        of normal:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyLess:
+          if d.commit == InvalidCommit:
+            return getLastTaggedCommit(c)
+          elif not commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyGreater:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            useNextOne = true
+          elif useNextOne:
+            return commitsAndTags[0]
+
+  return ""
+
+proc shortToCommit(c: var AtlasContext; short: string): string =
+  let (cc, status) = exec(c, GitRevParse, [short])
+  result = if status == 0: strutils.strip(cc) else: ""
+
+proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
+  let (_, status) = exec(c, GitCheckout, [commit])
+  if status != 0:
+    error(c, p, "could not checkout commit " & commit)
+
+proc gitPull(c: var AtlasContext; p: PackageName) =
+  let (_, status) = exec(c, GitPull, [])
+  if status != 0:
+    error(c, p, "could not 'git pull'")
+
+proc gitTag(c: var AtlasContext; tag: string) =
+  let (_, status) = exec(c, GitTag, [tag])
+  if status != 0:
+    error(c, c.projectDir.PackageName, "could not 'git tag " & tag & "'")
+
+proc pushTag(c: var AtlasContext; tag: string) =
+  let (outp, status) = exec(c, GitPush, [tag])
+  if status != 0:
+    error(c, c.projectDir.PackageName, "could not 'git push " & tag & "'")
+  elif outp.strip() == "Everything up-to-date":
+    info(c, c.projectDir.PackageName, "is up-to-date")
+  else:
+    info(c, c.projectDir.PackageName, "successfully pushed tag: " & tag)
+
+proc incrementTag(c: var AtlasContext; lastTag: string; field: Natural): string =
+  var startPos =
+    if lastTag[0] in {'0'..'9'}: 0
+    else: 1
+  var endPos = lastTag.find('.', startPos)
+  if field >= 1:
+    for i in 1 .. field:
+      if endPos == -1:
+        error c, projectFromCurrentDir(), "the last tag '" & lastTag & "' is missing . periods"
+        return ""
+      startPos = endPos + 1
+      endPos = lastTag.find('.', startPos)
+  if endPos == -1:
+    endPos = len(lastTag)
+  let patchNumber = parseInt(lastTag[startPos..<endPos])
+  lastTag[0..<startPos] & $(patchNumber + 1) & lastTag[endPos..^1]
+
+proc incrementLastTag(c: var AtlasContext; field: Natural): string =
+  let (ltr, status) = exec(c, GitLastTaggedRef, [])
+  if status == 0:
+    let
+      lastTaggedRef = ltr.strip()
+      lastTag = gitDescribeRefTag(c, lastTaggedRef)
+      currentCommit = exec(c, GitCurrentCommit, [])[0].strip()
+
+    if lastTaggedRef == currentCommit:
+      info c, c.projectDir.PackageName, "the current commit '" & currentCommit & "' is already tagged '" & lastTag & "'"
+      lastTag
+    else:
+      incrementTag(c, lastTag, field)
+  else: "v0.0.1" # assuming no tags have been made yet
+
+proc tag(c: var AtlasContext; tag: string) =
+  gitTag(c, tag)
+  pushTag(c, tag)
+
+proc tag(c: var AtlasContext; field: Natural) =
+  let oldErrors = c.errors
+  let newTag = incrementLastTag(c, field)
+  if c.errors == oldErrors:
+    tag(c, newTag)
+
+proc updatePackages(c: var AtlasContext) =
+  if dirExists(c.workspace / PackagesDir):
+    withDir(c, c.workspace / PackagesDir):
+      gitPull(c, PackageName PackagesDir)
+  else:
+    withDir c, c.workspace:
+      let err = cloneUrl(c, "https://github.com/nim-lang/packages", PackagesDir, false)
+      if err != "":
+        error c, PackageName(PackagesDir), err
+
+proc fillPackageLookupTable(c: var AtlasContext) =
+  if not c.hasPackageList:
+    c.hasPackageList = true
+    when not MockupRun:
+      updatePackages(c)
+    let plist = getPackages(when MockupRun: TestsDir else: c.workspace)
+    for entry in plist:
+      c.p[unicode.toLower entry.name] = entry.url
+
+proc toUrl(c: var AtlasContext; p: string): string =
+  if p.isUrl:
+    if c.usesOverrides:
+      result = c.overrides.substitute(p)
+      if result.len > 0: return result
+    result = p
+  else:
+    # either the project name or the URL can be overwritten!
+    if c.usesOverrides:
+      result = c.overrides.substitute(p)
+      if result.len > 0: return result
+
+    fillPackageLookupTable(c)
+    result = c.p.getOrDefault(unicode.toLower p)
+
+    if result.len == 0:
+      result = getUrlFromGithub(p)
+      if result.len == 0:
+        inc c.errors
+
+    if c.usesOverrides:
+      let newUrl = c.overrides.substitute(result)
+      if newUrl.len > 0: return newUrl
+
+proc toName(p: string): PackageName =
+  if p.isUrl:
+    result = PackageName splitFile(p).name
+  else:
+    result = PackageName p
+
+proc generateDepGraph(c: var AtlasContext; g: DepGraph) =
+  proc repr(w: Dependency): string = w.url / w.commit
+
+  var dotGraph = ""
+  for i in 0 ..< g.nodes.len:
+    dotGraph.addf("\"$1\" [label=\"$2\"];\n", [g.nodes[i].repr, if g.nodes[i].active: "" else: "unused"])
+  for i in 0 ..< g.nodes.len:
+    for p in items g.nodes[i].parents:
+      if p >= 0:
+        dotGraph.addf("\"$1\" -> \"$2\";\n", [g.nodes[p].repr, g.nodes[i].repr])
+  let dotFile = c.workspace / "deps.dot"
+  writeFile(dotFile, "digraph deps {\n$1}\n" % dotGraph)
+  let graphvizDotPath = findExe("dot")
+  if graphvizDotPath.len == 0:
+    #echo("gendepend: Graphviz's tool dot is required, " &
+    #  "see https://graphviz.org/download for downloading")
+    discard
+  else:
+    discard execShellCmd("dot -Tpng -odeps.png " & quoteShell(dotFile))
+
+proc showGraph(c: var AtlasContext; g: DepGraph) =
+  if c.showGraph:
+    generateDepGraph c, g
+
+proc needsCommitLookup(commit: string): bool {.inline.} =
+  '.' in commit or commit == InvalidCommit
+
+proc isShortCommitHash(commit: string): bool {.inline.} =
+  commit.len >= 4 and commit.len < 40
+
+proc getRequiredCommit(c: var AtlasContext; w: Dependency): string =
+  if needsCommitLookup(w.commit): versionToCommit(c, w)
+  elif isShortCommitHash(w.commit): shortToCommit(c, w.commit)
+  else: w.commit
+
+proc getRemoteUrl(): string =
+  execProcess("git config --get remote.origin.url").strip()
+
+proc genLockEntry(c: var AtlasContext; w: Dependency; dir: string) =
+  let url = getRemoteUrl()
+  var commit = getRequiredCommit(c, w)
+  if commit.len == 0 or needsCommitLookup(commit):
+    commit = execProcess("git log -1 --pretty=format:%H").strip()
+  c.lockFileToWrite.add LockFileEntry(dir: relativePath(dir, c.workspace, '/'), url: url, commit: commit)
+
+proc commitFromLockFile(c: var AtlasContext; dir: string): string =
+  let url = getRemoteUrl()
+  let d = relativePath(dir, c.workspace, '/')
+  if d in c.lockFileToUse:
+    result = c.lockFileToUse[d].commit
+    let wanted = c.lockFileToUse[d].url
+    if wanted != url:
+      error c, PackageName(d), "remote URL has been compromised: got: " &
+          url & " but wanted: " & wanted
+  else:
+    error c, PackageName(d), "package is not listed in the lock file"
+
+proc dependencyDir(c: AtlasContext; w: Dependency): string =
+  result = c.workspace / w.name.string
+  if not dirExists(result):
+    result = c.depsDir / w.name.string
+
+proc selectNode(c: var AtlasContext; g: var DepGraph; w: Dependency) =
+  # all other nodes of the same project name are not active
+  for e in items g.byName[w.name]:
+    g.nodes[e].active = e == w.self
+
+proc checkoutCommit(c: var AtlasContext; g: var DepGraph; w: Dependency) =
+  let dir = dependencyDir(c, w)
+  withDir c, dir:
+    if c.lockOption == genLock:
+      genLockEntry(c, w, dir)
+    elif c.lockOption == useLock:
+      checkoutGitCommit(c, w.name, commitFromLockFile(c, dir))
+    elif w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
+      gitPull(c, w.name)
+    else:
+      let err = isCleanGit(c)
+      if err != "":
+        warn c, w.name, err
+      else:
+        let requiredCommit = getRequiredCommit(c, w)
+        let (cc, status) = exec(c, GitCurrentCommit, [])
+        let currentCommit = strutils.strip(cc)
+        if requiredCommit == "" or status != 0:
+          if requiredCommit == "" and w.commit == InvalidCommit:
+            warn c, w.name, "package has no tagged releases"
+          else:
+            warn c, w.name, "cannot find specified version/commit " & w.commit
+        else:
+          if currentCommit != requiredCommit:
+            # checkout the later commit:
+            # git merge-base --is-ancestor <commit> <commit>
+            let (cc, status) = exec(c, GitMergeBase, [currentCommit, requiredCommit])
+            let mergeBase = strutils.strip(cc)
+            if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
+              # conflict resolution: pick the later commit:
+              if mergeBase == currentCommit:
+                checkoutGitCommit(c, w.name, requiredCommit)
+                selectNode c, g, w
+            else:
+              checkoutGitCommit(c, w.name, requiredCommit)
+              selectNode c, g, w
+              when false:
+                warn c, w.name, "do not know which commit is more recent:",
+                  currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
+
+proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
+  when MockupRun:
+    result = TestsDir / dep.name.string & ".nimble"
+    doAssert fileExists(result), "file does not exist " & result
+  else:
+    let dir = dependencyDir(c, dep)
+    result = dir / (dep.name.string & ".nimble")
+    if not fileExists(result):
+      result = ""
+      for x in walkFiles(dir / "*.nimble"):
+        if result.len == 0:
+          result = x
+        else:
+          # ambiguous .nimble file
+          return ""
+
+proc addUnique[T](s: var seq[T]; elem: sink T) =
+  if not s.contains(elem): s.add elem
+
+proc addUniqueDep(c: var AtlasContext; g: var DepGraph; parent: int;
+                  tokens: seq[string]; lockfile: Table[string, LockFileEntry]) =
+  let pkgName = tokens[0]
+  let oldErrors = c.errors
+  let url = toUrl(c, pkgName)
+  if oldErrors != c.errors:
+    warn c, toName(pkgName), "cannot resolve package name"
+  else:
+    let key = url / tokens[2]
+    if g.processed.hasKey(key):
+      g.nodes[g.processed[key]].parents.addUnique parent
+    else:
+      let self = g.nodes.len
+      g.byName.mgetOrPut(toName(pkgName), @[]).add self
+      g.processed[key] = self
+      if lockfile.contains(pkgName):
+        g.nodes.add Dependency(name: toName(pkgName),
+                            url: lockfile[pkgName].url,
+                            commit: lockfile[pkgName].commit,
+                            rel: normal,
+                            self: self,
+                            parents: @[parent])
+      else:
+        g.nodes.add Dependency(name: toName(pkgName), url: url, commit: tokens[2],
+                               rel: toDepRelation(tokens[1]),
+                               self: self,
+                               parents: @[parent])
+
+template toDestDir(p: PackageName): string = p.string
+
+proc readLockFile(filename: string): Table[string, LockFileEntry] =
+  let jsonAsStr = readFile(filename)
+  let jsonTree = parseJson(jsonAsStr)
+  let data = to(jsonTree, seq[LockFileEntry])
+  result = initTable[string, LockFileEntry]()
+  for d in items(data):
+    result[d.dir] = d
+
+proc collectDeps(c: var AtlasContext; g: var DepGraph; parent: int;
+                 dep: Dependency; nimbleFile: string): CfgPath =
+  # If there is a .nimble file, return the dependency path & srcDir
+  # else return "".
+  assert nimbleFile != ""
+  let nimbleInfo = extractRequiresInfo(c, nimbleFile)
+
+  let lockFilePath = dependencyDir(c, dep) / LockFileName
+  let lockFile = if fileExists(lockFilePath): readLockFile(lockFilePath)
+                 else: initTable[string, LockFileEntry]()
+
+  for r in nimbleInfo.requires:
+    var tokens: seq[string] = @[]
+    for token in tokenizeRequires(r):
+      tokens.add token
+    if tokens.len == 1:
+      # nimx uses dependencies like 'requires "sdl2"'.
+      # Via this hack we map them to the first tagged release.
+      # (See the `isStrictlySmallerThan` logic.)
+      tokens.add "<"
+      tokens.add InvalidCommit
+    elif tokens.len == 2 and tokens[1].startsWith("#"):
+      # Dependencies can also look like 'requires "sdl2#head"
+      var commit = tokens[1][1 .. ^1]
+      tokens[1] = "=="
+      tokens.add commit
+
+    if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0:
+      c.addUniqueDep g, parent, tokens, lockFile
+  result = CfgPath(toDestDir(dep.name) / nimbleInfo.srcDir)
+
+proc collectNewDeps(c: var AtlasContext; g: var DepGraph; parent: int;
+                    dep: Dependency): CfgPath =
+  let nimbleFile = findNimbleFile(c, dep)
+  if nimbleFile != "":
+    result = collectDeps(c, g, parent, dep, nimbleFile)
+  else:
+    result = CfgPath toDestDir(dep.name)
+
+proc selectDir(a, b: string): string = (if dirExists(a): a else: b)
+
+const
+  FileProtocol = "file://"
+  ThisVersion = "current_version.atlas"
+
+proc copyFromDisk(c: var AtlasContext; w: Dependency) =
+  let destDir = toDestDir(w.name)
+  var u = w.url.substr(FileProtocol.len)
+  if u.startsWith("./"): u = c.workspace / u.substr(2)
+  copyDir(selectDir(u & "@" & w.commit, u), destDir)
+  writeFile destDir / ThisVersion, w.commit
+  #echo "WRITTEN ", destDir / ThisVersion
+
+proc isNewerVersion(a, b: string): bool =
+  # only used for testing purposes.
+  if a == InvalidCommit or b == InvalidCommit:
+    return true
+  var amajor, aminor, apatch, bmajor, bminor, bpatch: int
+  if scanf(a, "$i.$i.$i", amajor, aminor, apatch):
+    assert scanf(b, "$i.$i.$i", bmajor, bminor, bpatch)
+    result = (amajor, aminor, apatch) > (bmajor, bminor, bpatch)
+  else:
+    assert scanf(a, "$i.$i", amajor, aminor)
+    assert scanf(b, "$i.$i", bmajor, bminor)
+    result = (amajor, aminor) > (bmajor, bminor)
+
+proc isLaterCommit(destDir, version: string): bool =
+  let oldVersion = try: readFile(destDir / ThisVersion).strip: except: "0.0"
+  result = isNewerVersion(version, oldVersion)
+
+proc traverseLoop(c: var AtlasContext; g: var DepGraph; startIsDep: bool): seq[CfgPath] =
+  result = @[]
+  var i = 0
+  while i < g.nodes.len:
+    let w = g.nodes[i]
+    let destDir = toDestDir(w.name)
+    let oldErrors = c.errors
+
+    if not dirExists(c.workspace / destDir) and not dirExists(c.depsDir / destDir):
+      withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
+        if w.url.startsWith(FileProtocol):
+          copyFromDisk c, w
+        else:
+          let err = cloneUrl(c, w.url, destDir, false)
+          if err != "":
+            error c, w.name, err
+    # assume this is the selected version, it might get overwritten later:
+    selectNode c, g, w
+    if oldErrors == c.errors:
+      if not c.keepCommits:
+        if not w.url.startsWith(FileProtocol):
+          checkoutCommit(c, g, w)
+        else:
+          withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
+            if isLaterCommit(destDir, w.commit):
+              copyFromDisk c, w
+              selectNode c, g, w
+      # even if the checkout fails, we can make use of the somewhat
+      # outdated .nimble file to clone more of the most likely still relevant
+      # dependencies:
+      result.addUnique collectNewDeps(c, g, i, w)
+    inc i
+
+proc traverse(c: var AtlasContext; start: string; startIsDep: bool): seq[CfgPath] =
+  # returns the list of paths for the nim.cfg file.
+  let url = toUrl(c, start)
+  var g = DepGraph(nodes: @[Dependency(name: toName(start), url: url, commit: "", self: 0)])
+  g.byName.mgetOrPut(toName(start), @[]).add 0
+
+  if url == "":
+    error c, toName(start), "cannot resolve package name"
+    return
+
+  c.projectDir = c.workspace / toDestDir(g.nodes[0].name)
+  if c.lockOption == useLock:
+    c.lockFileToUse = readLockFile(c.projectDir / LockFileName)
+  result = traverseLoop(c, g, startIsDep)
+  if c.lockOption == genLock:
+    writeFile c.projectDir / LockFileName, toJson(c.lockFileToWrite).pretty
+  showGraph c, g
+
+const
+  configPatternBegin = "############# begin Atlas config section ##########\n"
+  configPatternEnd =   "############# end Atlas config section   ##########\n"
+
+proc patchNimCfg(c: var AtlasContext; deps: seq[CfgPath]; cfgPath: string) =
+  var paths = "--noNimblePath\n"
+  for d in deps:
+    let pkgname = toDestDir d.string.PackageName
+    let pkgdir = if dirExists(c.workspace / pkgname): c.workspace / pkgname
+                 else: c.depsDir / pkgName
+    let x = relativePath(pkgdir, cfgPath, '/')
+    paths.add "--path:\"" & x & "\"\n"
+  var cfgContent = configPatternBegin & paths & configPatternEnd
+
+  when MockupRun:
+    assert readFile(TestsDir / "nim.cfg") == cfgContent
+    c.mockupSuccess = true
+  else:
+    let cfg = cfgPath / "nim.cfg"
+    assert cfgPath.len > 0
+    if cfgPath.len > 0 and not dirExists(cfgPath):
+      error(c, c.projectDir.PackageName, "could not write the nim.cfg")
+    elif not fileExists(cfg):
+      writeFile(cfg, cfgContent)
+      info(c, projectFromCurrentDir(), "created: " & cfg.readableFile)
+    else:
+      let content = readFile(cfg)
+      let start = content.find(configPatternBegin)
+      if start >= 0:
+        cfgContent = content.substr(0, start-1) & cfgContent
+        let theEnd = content.find(configPatternEnd, start)
+        if theEnd >= 0:
+          cfgContent.add content.substr(theEnd+len(configPatternEnd))
+      else:
+        cfgContent = content & "\n" & cfgContent
+      if cfgContent != content:
+        # do not touch the file if nothing changed
+        # (preserves the file date information):
+        writeFile(cfg, cfgContent)
+        info(c, projectFromCurrentDir(), "updated: " & cfg.readableFile)
+
+proc fatal*(msg: string) =
+  when defined(debug):
+    writeStackTrace()
+  quit "[Error] " & msg
+
+proc findSrcDir(c: var AtlasContext): string =
+  for nimbleFile in walkPattern(c.currentDir / "*.nimble"):
+    let nimbleInfo = extractRequiresInfo(c, nimbleFile)
+    return c.currentDir / nimbleInfo.srcDir
+  return c.currentDir
+
+proc installDependencies(c: var AtlasContext; nimbleFile: string; startIsDep: bool) =
+  # 1. find .nimble file in CWD
+  # 2. install deps from .nimble
+  var g = DepGraph(nodes: @[])
+  let (_, pkgname, _) = splitFile(nimbleFile)
+  let dep = Dependency(name: toName(pkgname), url: "", commit: "", self: 0)
+  discard collectDeps(c, g, -1, dep, nimbleFile)
+  let paths = traverseLoop(c, g, startIsDep)
+  patchNimCfg(c, paths, if c.cfgHere: c.currentDir else: findSrcDir(c))
+  showGraph c, g
+
+proc updateDir(c: var AtlasContext; dir, filter: string) =
+  for kind, file in walkDir(dir):
+    if kind == pcDir and dirExists(file / ".git"):
+      c.withDir file:
+        let pkg = PackageName(file)
+        let (remote, _) = osproc.execCmdEx("git remote -v")
+        if filter.len == 0 or filter in remote:
+          let diff = isCleanGit(c)
+          if diff != "":
+            warn(c, pkg, "has uncommitted changes; skipped")
+          else:
+            let (branch, _) = osproc.execCmdEx("git rev-parse --abbrev-ref HEAD")
+            if branch.strip.len > 0:
+              let (output, exitCode) = osproc.execCmdEx("git pull origin " & branch.strip)
+              if exitCode != 0:
+                error c, pkg, output
+              else:
+                info(c, pkg, "successfully updated")
+            else:
+              error c, pkg, "could not fetch current branch name"
+
+proc patchNimbleFile(c: var AtlasContext; dep: string): string =
+  let thisProject = c.currentDir.splitPath.tail
+  let oldErrors = c.errors
+  let url = toUrl(c, dep)
+  result = ""
+  if oldErrors != c.errors:
+    warn c, toName(dep), "cannot resolve package name"
+  else:
+    for x in walkFiles(c.currentDir / "*.nimble"):
+      if result.len == 0:
+        result = x
+      else:
+        # ambiguous .nimble file
+        warn c, toName(dep), "cannot determine `.nimble` file; there are multiple to choose from"
+        return ""
+    # see if we have this requirement already listed. If so, do nothing:
+    var found = false
+    if result.len > 0:
+      let nimbleInfo = extractRequiresInfo(c, result)
+      for r in nimbleInfo.requires:
+        var tokens: seq[string] = @[]
+        for token in tokenizeRequires(r):
+          tokens.add token
+        if tokens.len > 0:
+          let oldErrors = c.errors
+          let urlB = toUrl(c, tokens[0])
+          if oldErrors != c.errors:
+            warn c, toName(tokens[0]), "cannot resolve package name; found in: " & result
+          if url == urlB:
+            found = true
+            break
+
+    if not found:
+      let line = "requires \"$1\"\n" % dep.escape("", "")
+      if result.len > 0:
+        let oldContent = readFile(result)
+        writeFile result, oldContent & "\n" & line
+        info(c, toName(thisProject), "updated: " & result.readableFile)
+      else:
+        result = c.currentDir / thisProject & ".nimble"
+        writeFile result, line
+        info(c, toName(thisProject), "created: " & result.readableFile)
+    else:
+      info(c, toName(thisProject), "up to date: " & result.readableFile)
+
+proc detectWorkspace(currentDir: string): string =
+  result = currentDir
+  while result.len > 0:
+    if fileExists(result / AtlasWorkspace):
+      return result
+    result = result.parentDir()
+
+proc absoluteDepsDir(workspace, value: string): string =
+  if value == ".":
+    result = workspace
+  elif isAbsolute(value):
+    result = value
+  else:
+    result = workspace / value
+
+proc autoWorkspace(currentDir: string): string =
+  result = currentDir
+  while result.len > 0 and dirExists(result / ".git"):
+    result = result.parentDir()
+
+proc createWorkspaceIn(workspace, depsDir: string) =
+  if not fileExists(workspace / AtlasWorkspace):
+    writeFile workspace / AtlasWorkspace, "deps=\"$#\"" % escape(depsDir, "", "")
+  createDir absoluteDepsDir(workspace, depsDir)
+
+proc parseOverridesFile(c: var AtlasContext; filename: string) =
+  const Separator = " -> "
+  let path = c.workspace / filename
+  var f: File
+  if open(f, path):
+    c.usesOverrides = true
+    try:
+      var lineCount = 1
+      for line in lines(path):
+        let splitPos = line.find(Separator)
+        if splitPos >= 0 and line[0] != '#':
+          let key = line.substr(0, splitPos-1)
+          let val = line.substr(splitPos+len(Separator))
+          if key.len == 0 or val.len == 0:
+            error c, toName(path), "key/value must not be empty"
+          let err = c.overrides.addPattern(key, val)
+          if err.len > 0:
+            error c, toName(path), "(" & $lineCount & "): " & err
+        else:
+          discard "ignore the line"
+        inc lineCount
+    finally:
+      close f
+  else:
+    error c, toName(path), "cannot open: " & path
+
+proc readConfig(c: var AtlasContext) =
+  let configFile = c.workspace / AtlasWorkspace
+  var f = newFileStream(configFile, fmRead)
+  if f == nil:
+    error c, toName(configFile), "cannot open: " & configFile
+    return
+  var p: CfgParser
+  open(p, f, configFile)
+  while true:
+    var e = next(p)
+    case e.kind
+    of cfgEof: break
+    of cfgSectionStart:
+      discard "who cares about sections"
+    of cfgKeyValuePair:
+      case e.key.normalize
+      of "deps":
+        c.depsDir = absoluteDepsDir(c.workspace, e.value)
+      of "overrides":
+        parseOverridesFile(c, e.value)
+      else:
+        warn c, toName(configFile), "ignored unknown setting: " & e.key
+    of cfgOption:
+      discard "who cares about options"
+    of cfgError:
+      error c, toName(configFile), e.msg
+  close(p)
+
+const
+  BatchFile = """
+@echo off
+set PATH="$1";%PATH%
+"""
+  ShellFile = "export PATH=$1:$$PATH\n"
+
+proc setupNimEnv(c: var AtlasContext; nimVersion: string) =
+  template isDevel(nimVersion: string): bool = nimVersion == "devel"
+
+  template exec(c: var AtlasContext; command: string) =
+    let cmd = command # eval once
+    if os.execShellCmd(cmd) != 0:
+      error c, toName("nim-" & nimVersion), "failed: " & cmd
+      return
+
+  let nimDest = "nim-" & nimVersion
+  if dirExists(c.workspace / nimDest):
+    info c, toName(nimDest), "already exists; remove or rename and try again"
+    return
+
+  var major, minor, patch: int
+  if nimVersion != "devel":
+    if not scanf(nimVersion, "$i.$i.$i", major, minor, patch):
+      error c, toName("nim"), "cannot parse version requirement"
+      return
+  let csourcesVersion =
+    if nimVersion.isDevel or (major == 1 and minor >= 9) or major >= 2:
+      # already uses csources_v2
+      "csources_v2"
+    elif major == 0:
+      "csources" # has some chance of working
+    else:
+      "csources_v1"
+  withDir c, c.workspace:
+    if not dirExists(csourcesVersion):
+      exec c, "git clone https://github.com/nim-lang/" & csourcesVersion
+    exec c, "git clone https://github.com/nim-lang/nim " & nimDest
+  withDir c, c.workspace / csourcesVersion:
+    when defined(windows):
+      exec c, "build.bat"
+    else:
+      let makeExe = findExe("make")
+      if makeExe.len == 0:
+        exec c, "sh build.sh"
+      else:
+        exec c, "make"
+  let nimExe0 = ".." / csourcesVersion / "bin" / "nim".addFileExt(ExeExt)
+  withDir c, c.workspace / nimDest:
+    let nimExe = "bin" / "nim".addFileExt(ExeExt)
+    copyFileWithPermissions nimExe0, nimExe
+    let dep = Dependency(name: toName(nimDest), rel: normal, commit: nimVersion, self: 0)
+    if not nimVersion.isDevel:
+      let commit = versionToCommit(c, dep)
+      if commit.len == 0:
+        error c, toName(nimDest), "cannot resolve version to a commit"
+        return
+      checkoutGitCommit(c, dep.name, commit)
+    exec c, nimExe & " c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch"
+    let kochExe = when defined(windows): "koch.exe" else: "./koch"
+    exec c, kochExe & " boot -d:release --skipUserCfg --skipParentCfg --hints:off"
+    exec c, kochExe & " tools --skipUserCfg --skipParentCfg --hints:off"
+    # remove any old atlas binary that we now would end up using:
+    if cmpPaths(getAppDir(), c.workspace / nimDest / "bin") != 0:
+      removeFile "bin" / "atlas".addFileExt(ExeExt)
+    let pathEntry = (c.workspace / nimDest / "bin")
+    when defined(windows):
+      writeFile "activate.bat", BatchFile % pathEntry.replace('/', '\\')
+      info c, toName(nimDest), "RUN\nnim-" & nimVersion & "\\activate.bat"
+    else:
+      writeFile "activate.sh", ShellFile % pathEntry
+      info c, toName(nimDest), "RUN\nsource nim-" & nimVersion & "/activate.sh"
+
+proc main =
+  var action = ""
+  var args: seq[string] = @[]
+  template singleArg() =
+    if args.len != 1:
+      fatal action & " command takes a single package name"
+
+  template noArgs() =
+    if args.len != 0:
+      fatal action & " command takes no arguments"
+
+  template projectCmd() =
+    if c.projectDir == c.workspace or c.projectDir == c.depsDir:
+      fatal action & " command must be executed in a project, not in the workspace"
+
+  var c = AtlasContext(projectDir: getCurrentDir(), currentDir: getCurrentDir(), workspace: "")
+  var autoinit = false
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      if action.len == 0:
+        action = key.normalize
+      else:
+        args.add key
+    of cmdLongOption, cmdShortOption:
+      case normalize(key)
+      of "help", "h": writeHelp()
+      of "version", "v": writeVersion()
+      of "keepcommits": c.keepCommits = true
+      of "workspace":
+        if val == ".":
+          c.workspace = getCurrentDir()
+          createWorkspaceIn c.workspace, c.depsDir
+        elif val.len > 0:
+          c.workspace = val
+          createDir(val)
+          createWorkspaceIn c.workspace, c.depsDir
+        else:
+          writeHelp()
+      of "project":
+        if isAbsolute(val):
+          c.currentDir = val
+        else:
+          c.currentDir = getCurrentDir() / val
+      of "deps":
+        if val.len > 0:
+          c.depsDir = val
+        else:
+          writeHelp()
+      of "cfghere": c.cfgHere = true
+      of "autoinit": autoinit = true
+      of "showgraph": c.showGraph = true
+      of "genlock":
+        if c.lockOption != useLock:
+          c.lockOption = genLock
+        else:
+          writeHelp()
+      of "uselock":
+        if c.lockOption != genLock:
+          c.lockOption = useLock
+        else:
+          writeHelp()
+      of "colors":
+        case val.normalize
+        of "off": c.noColors = true
+        of "on": c.noColors = false
+        else: writeHelp()
+      else: writeHelp()
+    of cmdEnd: assert false, "cannot happen"
+
+  if c.workspace.len > 0:
+    if not dirExists(c.workspace): fatal "Workspace directory '" & c.workspace & "' not found."
+  elif action != "init":
+    when MockupRun:
+      c.workspace = autoWorkspace(c.currentDir)
+    else:
+      c.workspace = detectWorkspace(c.currentDir)
+      if c.workspace.len > 0:
+        readConfig c
+        info c, toName(c.workspace.readableFile), "is the current workspace"
+      elif autoinit:
+        c.workspace = autoWorkspace(c.currentDir)
+        createWorkspaceIn c.workspace, c.depsDir
+      elif action notin ["search", "list"]:
+        fatal "No workspace found. Run `atlas init` if you want this current directory to be your workspace."
+
+  when MockupRun:
+    c.depsDir = c.workspace
+
+  case action
+  of "":
+    fatal "No action."
+  of "init":
+    c.workspace = getCurrentDir()
+    createWorkspaceIn c.workspace, c.depsDir
+  of "clone", "update":
+    singleArg()
+    let deps = traverse(c, args[0], startIsDep = false)
+    patchNimCfg c, deps, if c.cfgHere: c.currentDir else: findSrcDir(c)
+    when MockupRun:
+      if not c.mockupSuccess:
+        fatal "There were problems."
+    else:
+      if c.errors > 0:
+        fatal "There were problems."
+  of "use":
+    projectCmd()
+    singleArg()
+    let nimbleFile = patchNimbleFile(c, args[0])
+    if nimbleFile.len > 0:
+      installDependencies(c, nimbleFile, startIsDep = false)
+  of "install":
+    projectCmd()
+    if args.len > 1:
+      fatal "install command takes a single argument"
+    var nimbleFile = ""
+    if args.len == 1:
+      nimbleFile = args[0]
+    else:
+      for x in walkPattern("*.nimble"):
+        nimbleFile = x
+        break
+    if nimbleFile.len == 0:
+      fatal "could not find a .nimble file"
+    else:
+      installDependencies(c, nimbleFile, startIsDep = true)
+  of "refresh":
+    noArgs()
+    updatePackages(c)
+  of "search", "list":
+    if c.workspace.len != 0:
+      updatePackages(c)
+      search getPackages(c.workspace), args
+    else: search @[], args
+  of "updateprojects":
+    updateDir(c, c.workspace, if args.len == 0: "" else: args[0])
+  of "updatedeps":
+    updateDir(c, c.depsDir, if args.len == 0: "" else: args[0])
+  of "extract":
+    singleArg()
+    if fileExists(args[0]):
+      echo toJson(extractRequiresInfo(args[0]))
+    else:
+      fatal "File does not exist: " & args[0]
+  of "tag":
+    projectCmd()
+    if args.len == 0:
+      tag(c, ord(patch))
+    elif args[0].len == 1 and args[0][0] in {'a'..'z'}:
+      let field = ord(args[0][0]) - ord('a')
+      tag(c, field)
+    elif args[0].len == 1 and args[0][0] in {'A'..'Z'}:
+      let field = ord(args[0][0]) - ord('A')
+      tag(c, field)
+    elif '.' in args[0]:
+      tag(c, args[0])
+    else:
+      var field: SemVerField
+      try: field = parseEnum[SemVerField](args[0])
+      except: fatal "tag command takes one of 'patch' 'minor' 'major', a SemVer tag, or a letter from 'a' to 'z'"
+      tag(c, ord(field))
+  of "build", "test", "doc", "tasks":
+    projectCmd()
+    nimbleExec(action, args)
+  of "task":
+    projectCmd()
+    nimbleExec("", args)
+  of "env":
+    singleArg()
+    setupNimEnv c, args[0]
+  else:
+    fatal "Invalid action: " & action
+
+when isMainModule:
+  main()
diff --git a/atlas/atlas.nim.cfg b/atlas/atlas.nim.cfg
new file mode 100644
index 000000000..fcace0579
--- /dev/null
+++ b/atlas/atlas.nim.cfg
@@ -0,0 +1 @@
+--define:ssl

diff --git a/atlas/compiledpatterns.nim b/atlas/compiledpatterns.nim
new file mode 100644
index 000000000..69751d82b
--- /dev/null
+++ b/atlas/compiledpatterns.nim
@@ -0,0 +1,246 @@
+#
+#           Atlas Package Cloner
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+##[
+
+Syntax taken from strscans.nim:
+
+=================   ========================================================
+``$$``              Matches a single dollar sign.
+``$*``              Matches until the token following the ``$*`` was found.
+                    The match is allowed to be of 0 length.
+``$+``              Matches until the token following the ``$+`` was found.
+                    The match must consist of at least one char.
+``$s``              Skips optional whitespace.
+=================   ========================================================
+
+]##
+
+import tables
+from strutils import continuesWith, Whitespace
+
+type
+  Opcode = enum
+    MatchVerbatim # needs verbatim match
+    Capture0Until
+    Capture1Until
+    Capture0UntilEnd
+    Capture1UntilEnd
+    SkipWhitespace
+
+  Instr = object
+    opc: Opcode
+    arg1: uint8
+    arg2: uint16
+
+  Pattern* = object
+    code: seq[Instr]
+    usedMatches: int
+    error: string
+
+# A rewrite rule looks like:
+#
+# foo$*bar -> https://gitlab.cross.de/$1
+
+proc compile*(pattern: string; strings: var seq[string]): Pattern =
+  proc parseSuffix(s: string; start: int): int =
+    result = start
+    while result < s.len and s[result] != '$':
+      inc result
+
+  result = Pattern(code: @[], usedMatches: 0, error: "")
+  var p = 0
+  while p < pattern.len:
+    if pattern[p] == '$' and p+1 < pattern.len:
+      case pattern[p+1]
+      of '$':
+        if result.code.len > 0 and result.code[^1].opc in {
+              MatchVerbatim, Capture0Until, Capture1Until, Capture0UntilEnd, Capture1UntilEnd}:
+          # merge with previous opcode
+          let key = strings[result.code[^1].arg2] & "$"
+          var idx = find(strings, key)
+          if idx < 0:
+            idx = strings.len
+            strings.add key
+          result.code[^1].arg2 = uint16(idx)
+        else:
+          var idx = find(strings, "$")
+          if idx < 0:
+            idx = strings.len
+            strings.add "$"
+          result.code.add Instr(opc: MatchVerbatim,
+                                arg1: uint8(0), arg2: uint16(idx))
+        inc p, 2
+      of '+', '*':
+        let isPlus = pattern[p+1] == '+'
+
+        let pEnd = parseSuffix(pattern, p+2)
+        let suffix = pattern.substr(p+2, pEnd-1)
+        p = pEnd
+        if suffix.len == 0:
+          result.code.add Instr(opc: if isPlus: Capture1UntilEnd else: Capture0UntilEnd,
+                                arg1: uint8(result.usedMatches), arg2: uint16(0))
+        else:
+          var idx = find(strings, suffix)
+          if idx < 0:
+            idx = strings.len
+            strings.add suffix
+          result.code.add Instr(opc: if isPlus: Capture1Until else: Capture0Until,
+                                arg1: uint8(result.usedMatches), arg2: uint16(idx))
+        inc result.usedMatches
+
+      of 's':
+        result.code.add Instr(opc: SkipWhitespace)
+        inc p, 2
+      else:
+        result.error = "unknown syntax '$" & pattern[p+1] & "'"
+        break
+    elif pattern[p] == '$':
+      result.error = "unescaped '$'"
+      break
+    else:
+      let pEnd = parseSuffix(pattern, p)
+      let suffix = pattern.substr(p, pEnd-1)
+      var idx = find(strings, suffix)
+      if idx < 0:
+        idx = strings.len
+        strings.add suffix
+      result.code.add Instr(opc: MatchVerbatim,
+                            arg1: uint8(0), arg2: uint16(idx))
+      p = pEnd
+
+type
+  MatchObj = object
+    m: int
+    a: array[20, (int, int)]
+
+proc matches(s: Pattern; strings: seq[string]; input: string): MatchObj =
+  template failed =
+    result.m = -1
+    return result
+
+  var i = 0
+  for instr in s.code:
+    case instr.opc
+    of MatchVerbatim:
+      if continuesWith(input, strings[instr.arg2], i):
+        inc i, strings[instr.arg2].len
+      else:
+        failed()
+    of Capture0Until, Capture1Until:
+      block searchLoop:
+        let start = i
+        while i < input.len:
+          if continuesWith(input, strings[instr.arg2], i):
+            if instr.opc == Capture1Until and i == start:
+              failed()
+            result.a[result.m] = (start, i-1)
+            inc result.m
+            inc i, strings[instr.arg2].len
+            break searchLoop
+          inc i
+        failed()
+
+    of Capture0UntilEnd, Capture1UntilEnd:
+      if instr.opc == Capture1UntilEnd and i >= input.len:
+        failed()
+      result.a[result.m] = (i, input.len-1)
+      inc result.m
+      i = input.len
+    of SkipWhitespace:
+      while i < input.len and input[i] in Whitespace: inc i
+  if i < input.len:
+    # still unmatched stuff was left:
+    failed()
+
+proc translate(m: MatchObj; outputPattern, input: string): string =
+  result = newStringOfCap(outputPattern.len)
+  var i = 0
+  var patternCount = 0
+  while i < outputPattern.len:
+    if i+1 < outputPattern.len and outputPattern[i] == '$':
+      if outputPattern[i+1] == '#':
+        inc i, 2
+        if patternCount < m.a.len:
+          let (a, b) = m.a[patternCount]
+          for j in a..b: result.add input[j]
+        inc patternCount
+      elif outputPattern[i+1] in {'1'..'9'}:
+        var n = ord(outputPattern[i+1]) - ord('0')
+        inc i, 2
+        while i < outputPattern.len and outputPattern[i] in {'0'..'9'}:
+          n = n * 10 + (ord(outputPattern[i]) - ord('0'))
+          inc i
+        patternCount = n
+        if n-1 < m.a.len:
+          let (a, b) = m.a[n-1]
+          for j in a..b: result.add input[j]
+      else:
+        # just ignore the wrong pattern:
+        inc i
+    else:
+      result.add outputPattern[i]
+      inc i
+
+proc replace*(s: Pattern; outputPattern, input: string): string =
+  var strings: seq[string] = @[]
+  let m = s.matches(strings, input)
+  if m.m < 0:
+    result = ""
+  else:
+    result = translate(m, outputPattern, input)
+
+
+type
+  Patterns* = object
+    s: seq[(Pattern, string)]
+    t: Table[string, string]
+    strings: seq[string]
+
+proc initPatterns*(): Patterns =
+  Patterns(s: @[], t: initTable[string, string](), strings: @[])
+
+proc addPattern*(p: var Patterns; inputPattern, outputPattern: string): string =
+  if '$' notin inputPattern and '$' notin outputPattern:
+    p.t[inputPattern] = outputPattern
+    result = ""
+  else:
+    let code = compile(inputPattern, p.strings)
+    if code.error.len > 0:
+      result = code.error
+    else:
+      p.s.add (code, outputPattern)
+      result = ""
+
+proc substitute*(p: Patterns; input: string): string =
+  result = p.t.getOrDefault(input)
+  if result.len == 0:
+    for i in 0..<p.s.len:
+      let m = p.s[i][0].matches(p.strings, input)
+      if m.m >= 0:
+        return translate(m, p.s[i][1], input)
+
+proc replacePattern*(inputPattern, outputPattern, input: string): string =
+  var strings: seq[string] = @[]
+  let code = compile(inputPattern, strings)
+  result = replace(code, outputPattern, input)
+
+when isMainModule:
+  # foo$*bar -> https://gitlab.cross.de/$1
+  const realInput = "$fooXXbar$z00end"
+  var strings: seq[string] = @[]
+  let code = compile("$$foo$*bar$$$*z00$*", strings)
+  echo code
+
+  let m = code.matches(strings, realInput)
+  echo m.m
+
+  echo translate(m, "$1--$#-$#-", realInput)
+
+  echo translate(m, "https://gitlab.cross.de/$1", realInput)
+
diff --git a/atlas/osutils.nim b/atlas/osutils.nim
new file mode 100644
index 000000000..66cd29be5
--- /dev/null
+++ b/atlas/osutils.nim
@@ -0,0 +1,51 @@
+## OS utilities like 'withDir'.
+## (c) 2021 Andreas Rumpf
+
+import os, strutils, osproc
+
+proc isUrl*(x: string): bool =
+  x.startsWith("git://") or x.startsWith("https://") or x.startsWith("http://")
+
+proc cloneUrl*(url, dest: string; cloneUsingHttps: bool): string =
+  ## Returns an error message on error or else "".
+  result = ""
+  var modUrl =
+    if url.startsWith("git://") and cloneUsingHttps:
+      "https://" & url[6 .. ^1]
+    else: url
+
+  # github + https + trailing url slash causes a
+  # checkout/ls-remote to fail with Repository not found
+  var isGithub = false
+  if modUrl.contains("github.com") and modUrl.endsWith("/"):
+    modUrl = modUrl[0 .. ^2]
+    isGithub = true
+
+  let (_, exitCode) = execCmdEx("git ls-remote --quiet --tags " & modUrl)
+  var xcode = exitCode
+  if isGithub and exitCode != QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    for i in 0..4:
+      os.sleep(4000)
+      xcode = execCmdEx("git ls-remote --quiet --tags " & modUrl)[1]
+      if xcode == QuitSuccess: break
+
+  if xcode == QuitSuccess:
+    # retry multiple times to avoid annoying github timeouts:
+    let cmd = "git clone --recursive " & modUrl & " " & dest
+    for i in 0..4:
+      if execShellCmd(cmd) == 0: return ""
+      os.sleep(4000)
+    result = "exernal program failed: " & cmd
+  elif not isGithub:
+    let (_, exitCode) = execCmdEx("hg identify " & modUrl)
+    if exitCode == QuitSuccess:
+      let cmd = "hg clone " & modUrl & " " & dest
+      for i in 0..4:
+        if execShellCmd(cmd) == 0: return ""
+        os.sleep(4000)
+      result = "exernal program failed: " & cmd
+    else:
+      result = "Unable to identify url: " & modUrl
+  else:
+    result = "Unable to identify url: " & modUrl
diff --git a/atlas/packagesjson.nim b/atlas/packagesjson.nim
new file mode 100644
index 000000000..7e25c6934
--- /dev/null
+++ b/atlas/packagesjson.nim
@@ -0,0 +1,161 @@
+
+import std / [json, os, sets, strutils, httpclient, uri]
+
+type
+  Package* = ref object
+    # Required fields in a package.
+    name*: string
+    url*: string # Download location.
+    license*: string
+    downloadMethod*: string
+    description*: string
+    tags*: seq[string] # \
+    # From here on, optional fields set to the empty string if not available.
+    version*: string
+    dvcsTag*: string
+    web*: string # Info url for humans.
+
+proc optionalField(obj: JsonNode, name: string, default = ""): string =
+  if hasKey(obj, name) and obj[name].kind == JString:
+    result = obj[name].str
+  else:
+    result = default
+
+proc requiredField(obj: JsonNode, name: string): string =
+  result = optionalField(obj, name, "")
+
+proc fromJson*(obj: JSonNode): Package =
+  result = Package()
+  result.name = obj.requiredField("name")
+  if result.name.len == 0: return nil
+  result.version = obj.optionalField("version")
+  result.url = obj.requiredField("url")
+  if result.url.len == 0: return nil
+  result.downloadMethod = obj.requiredField("method")
+  if result.downloadMethod.len == 0: return nil
+  result.dvcsTag = obj.optionalField("dvcs-tag")
+  result.license = obj.optionalField("license")
+  result.tags = @[]
+  for t in obj["tags"]:
+    result.tags.add(t.str)
+  result.description = obj.requiredField("description")
+  result.web = obj.optionalField("web")
+
+const PackagesDir* = "packages"
+
+proc getPackages*(workspaceDir: string): seq[Package] =
+  result = @[]
+  var uniqueNames = initHashSet[string]()
+  var jsonFiles = 0
+  for kind, path in walkDir(workspaceDir / PackagesDir):
+    if kind == pcFile and path.endsWith(".json"):
+      inc jsonFiles
+      let packages = json.parseFile(path)
+      for p in packages:
+        let pkg = p.fromJson()
+        if pkg != nil and not uniqueNames.containsOrIncl(pkg.name):
+          result.add(pkg)
+
+proc `$`*(pkg: Package): string =
+  result = pkg.name & ":\n"
+  result &= "  url:         " & pkg.url & " (" & pkg.downloadMethod & ")\n"
+  result &= "  tags:        " & pkg.tags.join(", ") & "\n"
+  result &= "  description: " & pkg.description & "\n"
+  result &= "  license:     " & pkg.license & "\n"
+  if pkg.web.len > 0:
+    result &= "  website:     " & pkg.web & "\n"
+
+proc toTags(j: JsonNode): seq[string] =
+  result = @[]
+  if j.kind == JArray:
+    for elem in items j:
+      result.add elem.getStr("")
+
+proc singleGithubSearch(term: string): JsonNode =
+  # For example:
+  # https://api.github.com/search/repositories?q=weave+language:nim
+  var client = newHttpClient()
+  try:
+    let x = client.getContent("https://api.github.com/search/repositories?q=" & encodeUrl(term) & "+language:nim")
+    result = parseJson(x)
+  except:
+    result = parseJson("{\"items\": []}")
+  finally:
+    client.close()
+
+proc githubSearch(seen: var HashSet[string]; terms: seq[string]) =
+  for term in terms:
+    let results = singleGithubSearch(term)
+    for j in items(results.getOrDefault("items")):
+      let p = Package(
+        name: j.getOrDefault("name").getStr,
+        url: j.getOrDefault("html_url").getStr,
+        downloadMethod: "git",
+        tags: toTags(j.getOrDefault("topics")),
+        description: j.getOrDefault("description").getStr,
+        license: j.getOrDefault("license").getOrDefault("spdx_id").getStr,
+        web: j.getOrDefault("html_url").getStr
+      )
+      if not seen.containsOrIncl(p.url):
+        echo p
+
+proc getUrlFromGithub*(term: string): string =
+  let results = singleGithubSearch(term)
+  var matches = 0
+  result = ""
+  for j in items(results.getOrDefault("items")):
+    if cmpIgnoreCase(j.getOrDefault("name").getStr, term) == 0:
+      if matches == 0:
+        result = j.getOrDefault("html_url").getStr
+      inc matches
+  if matches != 1:
+    # ambiguous, not ok!
+    result = ""
+
+proc search*(pkgList: seq[Package]; terms: seq[string]) =
+  var seen = initHashSet[string]()
+  template onFound =
+    echo pkg
+    seen.incl pkg.url
+    break forPackage
+
+  for pkg in pkgList:
+    if terms.len > 0:
+      block forPackage:
+        for term in terms:
+          let word = term.toLower
+          # Search by name.
+          if word in pkg.name.toLower:
+            onFound()
+          # Search by tag.
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              onFound()
+    else:
+      echo(pkg)
+  githubSearch seen, terms
+  if seen.len == 0 and terms.len > 0:
+    echo("No package found.")
+
+type PkgCandidates* = array[3, seq[Package]]
+
+proc determineCandidates*(pkgList: seq[Package];
+                         terms: seq[string]): PkgCandidates =
+  result[0] = @[]
+  result[1] = @[]
+  result[2] = @[]
+  for pkg in pkgList:
+    block termLoop:
+      for term in terms:
+        let word = term.toLower
+        if word == pkg.name.toLower:
+          result[0].add pkg
+          break termLoop
+        elif word in pkg.name.toLower:
+          result[1].add pkg
+          break termLoop
+        else:
+          for tag in pkg.tags:
+            if word in tag.toLower:
+              result[2].add pkg
+              break termLoop
diff --git a/atlas/parse_requires.nim b/atlas/parse_requires.nim
new file mode 100644
index 000000000..7e26a1656
--- /dev/null
+++ b/atlas/parse_requires.nim
@@ -0,0 +1,101 @@
+## Utility API for Nim package managers.
+## (c) 2021 Andreas Rumpf
+
+import std / strutils
+import ".." / compiler / [ast, idents, msgs, syntaxes, options, pathutils]
+
+type
+  NimbleFileInfo* = object
+    requires*: seq[string]
+    srcDir*: string
+    tasks*: seq[(string, string)]
+
+proc extract(n: PNode; conf: ConfigRef; result: var NimbleFileInfo) =
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    for child in n:
+      extract(child, conf, result)
+  of nkCallKinds:
+    if n[0].kind == nkIdent:
+      case n[0].ident.s
+      of "requires":
+        for i in 1..<n.len:
+          var ch = n[i]
+          while ch.kind in {nkStmtListExpr, nkStmtList} and ch.len > 0: ch = ch.lastSon
+          if ch.kind in {nkStrLit..nkTripleStrLit}:
+            result.requires.add ch.strVal
+          else:
+            localError(conf, ch.info, "'requires' takes string literals")
+      of "task":
+        if n.len >= 3 and n[1].kind == nkIdent and n[2].kind in {nkStrLit..nkTripleStrLit}:
+          result.tasks.add((n[1].ident.s, n[2].strVal))
+      else: discard
+  of nkAsgn, nkFastAsgn:
+    if n[0].kind == nkIdent and cmpIgnoreCase(n[0].ident.s, "srcDir") == 0:
+      if n[1].kind in {nkStrLit..nkTripleStrLit}:
+        result.srcDir = n[1].strVal
+      else:
+        localError(conf, n[1].info, "assignments to 'srcDir' must be string literals")
+  else:
+    discard
+
+proc extractRequiresInfo*(nimbleFile: string): NimbleFileInfo =
+  ## Extract the `requires` information from a Nimble file. This does **not**
+  ## evaluate the Nimble file. Errors are produced on stderr/stdout and are
+  ## formatted as the Nim compiler does it. The parser uses the Nim compiler
+  ## as an API. The result can be empty, this is not an error, only parsing
+  ## errors are reported.
+  var conf = newConfigRef()
+  conf.foreignPackageNotes = {}
+  conf.notes = {}
+  conf.mainPackageNotes = {}
+
+  let fileIdx = fileInfoIdx(conf, AbsoluteFile nimbleFile)
+  var parser: Parser
+  if setupParser(parser, fileIdx, newIdentCache(), conf):
+    extract(parseAll(parser), conf, result)
+    closeParser(parser)
+
+const Operators* = {'<', '>', '=', '&', '@', '!', '^'}
+
+proc token(s: string; idx: int; lit: var string): int =
+  var i = idx
+  if i >= s.len: return i
+  while s[i] in Whitespace: inc(i)
+  case s[i]
+  of Letters, '#':
+    lit.add s[i]
+    inc i
+    while i < s.len and s[i] notin (Whitespace + {'@', '#'}):
+      lit.add s[i]
+      inc i
+  of '0'..'9':
+    while i < s.len and s[i] in {'0'..'9', '.'}:
+      lit.add s[i]
+      inc i
+  of '"':
+    inc i
+    while i < s.len and s[i] != '"':
+      lit.add s[i]
+      inc i
+    inc i
+  of Operators:
+    while i < s.len and s[i] in Operators:
+      lit.add s[i]
+      inc i
+  else:
+    lit.add s[i]
+    inc i
+  result = i
+
+iterator tokenizeRequires*(s: string): string =
+  var start = 0
+  var tok = ""
+  while start < s.len:
+    tok.setLen 0
+    start = token(s, start, tok)
+    yield tok
+
+when isMainModule:
+  for x in tokenizeRequires("jester@#head >= 1.5 & <= 1.8"):
+    echo x
diff --git a/atlas/testdata.nim b/atlas/testdata.nim
new file mode 100644
index 000000000..aefaeacd2
--- /dev/null
+++ b/atlas/testdata.nim
@@ -0,0 +1,63 @@
+
+type
+  PerDirData = object
+    dirname: string
+    cmd: Command
+    exitCode: int
+    output: string
+
+template toData(a, b, c, d): untyped =
+  PerDirData(dirname: a, cmd: b, exitCode: c, output: d)
+
+const
+  TestLog = [
+    toData("balls", GitPull, 0, "Already up to date.\n"),
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitMergeBase, 0, "349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n1b5643d04fba6d996a16d1ffc13d034a40003f8f\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "1b5643d04fba6d996a16d1ffc13d034a40003f8f"), # watch out!
+
+    toData("ups", GitDiff, 0, ""),
+    toData("ups", GitTags, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1 refs/tags/0.0.2\n19bc490c22b4f5b0628c31cdedead1375b279356 refs/tags/0.0.3\nff34602aaea824cb46d6588cd5fe1178132e9702 refs/tags/0.0.4\n09de599138f20b745133b6e4fe563e204415a7e8 refs/tags/0.0.5\n85fee3b74798311108a105635df31f892150f5d0 refs/tags/0.0.6\nfd303913b22b121dc42f332109e9c44950b9acd4 refs/tags/0.0.7\n"),
+    toData("ups", GitCurrentCommit, 0, "74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitMergeBase, 0, "74c31af8030112dac758440aa51ef175992f71f3\n4008f9339cd22b30e180bc87a6cca7270fd28ac1\n74c31af8030112dac758440aa51ef175992f71f3\n"),
+    toData("ups", GitCheckout, 0, "4008f9339cd22b30e180bc87a6cca7270fd28ac1"),
+
+    toData("sync", GitDiff, 0, ""),
+    toData("sync", GitRevParse, 0, "810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"),
+    toData("sync", GitCurrentCommit, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n"),
+    toData("sync", GitMergeBase, 0, "de5c7337ebc22422190e8aeca37d05651735f440\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n810bd2d75e9f6e182534ae2488670b51a9f13fc3\n"),
+
+    toData("npeg", GitDiff, 0, ""),
+    toData("npeg", GitTags, 0, "8df2f0c9391995fd086b8aab00e8ab7aded1e8f0 refs/tags/0.1.0\n4c959a72db5283b55eeef491076eefb5e02316f1 refs/tags/0.10.0\n802f47c0f7f4318a4f0858ba5a6a6ed2333bde71 refs/tags/0.11.0\n82c8d92837108dce225358ace2c416bf9a3f30ce refs/tags/0.12.0\n87d2f2c4f6ef7da350d45beb5a336611bde7f518 refs/tags/0.13.0\n39964f0d220bfaade47a568bf03c1cf28aa2bc37 refs/tags/0.14.0\nbe9f03f92304cbeab70572944a8563db9b23b2fb refs/tags/0.14.1\na933fb9832566fc95273e417597bfb4faf564ca6 refs/tags/0.15.0\n6aad2e438c52ff0636c7bfb64338e444ac3e83ba refs/tags/0.16.0\nf4ddffb5848c42c6151743dd9c7eddcaaabc56cc refs/tags/0.17.0\n30b446b39442cdbc53a97018ab8a54149aa7c3b7 refs/tags/0.17.1\n1a9d36aa3b34a6169d4530463f1c17a3fe1e075e refs/tags/0.18.0\ndd34f903a9a63b876cb2db19b7a4ce0bcc252134 refs/tags/0.19.0\nd93d49c81fc8722d7929ac463b435c0f2e10c53b refs/tags/0.2.0\neeae7746c9b1118bcf27744ab2aee26969051256 refs/tags/0.20.0\n8c3471a548129f3bf62df15cd0fd8cca1787d852 refs/tags/0.21.0\nc0e873a17bc713c80e74fec3c30cb62dcd5d194a refs/tags/0.21.1\nbae84c47a1bb259b209b6f6be1582327b784539d refs/tags/0.21.2\nbfcb4bcae76a917c3c88736ca773e4cb67dbb2d8 refs/tags/0.21.3\n0eabb7c462d30932049f0b7e6a030c1562cf9fee refs/tags/0.22.0\n2e75367095f54d4351005078bad98041a55b14c1 refs/tags/0.22.1\n814ea235dd398108d7b18f966694c3d951575701 refs/tags/0.22.2\na812064587d983c129737f8500bf74990e6b8dab refs/tags/0.23.0\nbd969ad3745db0d66022564cac76cf9424651104 refs/tags/0.23.1\na037c646a47623b92718efadc2bb74d03664b360 refs/tags/0.23.2\n078475ccceeaca0fac947492acdd24514da8d863 refs/tags/0.24.0\ne7bd87dc992512fd5825a557a56907647e03c979 refs/tags/0.24.1\n45ea601e1c7f64fb857bc99df984b86673621d2c refs/tags/0.3.0\n1ea9868a3fee3aa487ab7ec9129208a4dd483d0d refs/tags/0.4.0\n39afdb5733d3245386d29d08c5ff61c89268f499 refs/tags/0.5.0\n458c7b5910fcb157af3fc51bc3b3e663fdb3ed4a refs/tags/0.6.0\n06c38bd8563d822455bc237c2a98c153d938ed1b refs/tags/0.7.0\nf446b6056eef6d8dc9d8b47a79aca93d17dc8230 refs/tags/0.8.0\nbb25a195133f9f7af06386d0809793923cc5e8ab refs/tags/0.9.0\n"),
+    toData("npeg", GitCurrentCommit, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\n"),
+    toData("npeg", GitMergeBase, 0, "5d80f93aa720898936668b3bc47d0fff101ec414\na037c646a47623b92718efadc2bb74d03664b360\na037c646a47623b92718efadc2bb74d03664b360\n"),
+
+    toData("testes", GitDiff, 0, ""),
+    toData("testes", GitTags, 0, "3ce9b2968b5f644755a0ced1baa3eece88c2f12e refs/tags/0.1.0\nf73af8318b54737678fab8b54bdcd8a451015e0d refs/tags/0.1.1\nd21d84d37b161a123a43318bae353108755916de refs/tags/0.1.2\n5c36b6095353ed03b08ac939d00aff2d73f79a35 refs/tags/0.1.3\na1220d11237ee8f135f772ff9731c11b2d91ba31 refs/tags/0.1.4\n574f741b90d04a7ce8c9b990e6077708d7ad076e refs/tags/0.1.5\nced0a9e58234b680def6931578e09165a32e6291 refs/tags/0.1.6\nbb248952e8742a6011eb1a45a9d2059aeb0341d7 refs/tags/0.1.7\nabb7d7c552da0a8e0ddc586c15ccf7e74b0d068b refs/tags/0.10.0\n6e42a768a90d6442196b344bcdcb6f834b76e7b7 refs/tags/0.2.0\n9d136c3a0851ca2c021f5fb4f7b63f0a0ef77232 refs/tags/0.2.1\ndcb282b2da863fd2939e1969cec7a99788feb456 refs/tags/0.2.2\nf708a632afaa40a322a1a61c1c13722edac8e8c5 refs/tags/0.3.0\n3213f59e3f9ba052452c59f01d1418360d856af6 refs/tags/0.3.1\nf7bb1743dffd327958dfcebae4cfb6f61cc1cb8c refs/tags/0.3.2\n6b64569ebecad6bc60cc8697713701e7659204f4 refs/tags/0.3.3\nb51c25a4367bd17f419f78cb5a27f319e9d820f5 refs/tags/0.3.4\nb265612710cbd5ddb1b173c94ece8ec5c7ceccac refs/tags/0.3.5\ne404bcfe42e92d7509717a2dfa115cacb4964c5d refs/tags/0.3.6\n5e4d0d5b7e7f314dde701c546c4365c59782d3dc refs/tags/0.3.7\ne13f91c9c913d2b81c59adeaad687efa2b35293a refs/tags/0.3.8\n17599625f09af0ae4b525e63ab726a3002540702 refs/tags/0.3.9\n13e907f70571dd146d8dc29ddec4599b40ba4e85 refs/tags/0.4.0\n155a74cf676495df1e0674dd07b5e4a0291a9a4a refs/tags/0.4.1\nf37abccdc148cb02ca637a6f0bc8821491cce358 refs/tags/0.4.2\n0250d29ebdd02f28f9020445adb5a4e51fd1902c refs/tags/0.5.0\n2fb87db6d9f34109a70205876030c53f815739b7 refs/tags/0.5.1\n629d17ba8d6a1a4eca8145eb089ed5bca4473dfc refs/tags/0.6.0\ne926130f5f1b7903f68be49cc1563225bd9d948d refs/tags/0.7.0\n7365303897e6185796c274425c079916047e3f14 refs/tags/0.7.1\na735c4adabeba637409f41c4325dd8fc5fb91e2d refs/tags/0.7.10\nfe023fd27404889c5122f902456cbba14b767405 refs/tags/0.7.11\n4430e72972c77a5e9c1555d59bba11d840682691 refs/tags/0.7.12\nf0e53eb490a9558c7f594d2e095b70665e36ca88 refs/tags/0.7.13\nf6520e25e7c329c2957cda447f149fc6a930db0d refs/tags/0.7.2\nd509762f7191757c240d3c79c9ecda53f8c0cfe3 refs/tags/0.7.3\nc02e7a783d1c42fd1f91bca7142f7c3733950c05 refs/tags/0.7.4\n8c8a9e496e9b86ba7602709438980ca31e6989d9 refs/tags/0.7.5\n29839c18b4ac83c0111a178322b57ebb8a8d402c refs/tags/0.7.6\n3b62973cf74fafd8ea906644d89ac34d29a8a6cf refs/tags/0.7.7\ne67ff99dc43c391e89a37f97a9d298c3428bbde2 refs/tags/0.7.8\n4b72ecda0d40ed8e5ab8ad4095a0691d30ec6cd0 refs/tags/0.7.9\n2512b8cc3d7f001d277e89978da2049a5feee5c4 refs/tags/0.8.0\n86c47029690bd2731d204245f3f54462227bba0d refs/tags/0.9.0\n9a7f94f78588e9b5ba7ca077e1f7eae0607c6cf6 refs/tags/0.9.1\n08c915dc016d16c1dfa9a77d0b045ec29c9f2074 refs/tags/0.9.2\n3fb658b1ce1e1efa37d6f9f14322bdac8def02a5 refs/tags/0.9.3\n738fda0add962379ffe6aa6ca5f01a6943a98a2e refs/tags/0.9.4\n48d821add361f7ad768ecb35a0b19c38f90c919e refs/tags/0.9.5\nff9ae890f597dac301b2ac6e6805eb9ac5afd49a refs/tags/0.9.6\n483c78f06e60b0ec5e79fc3476df075ee7286890 refs/tags/0.9.7\n416eec87a5ae39a1a6035552e9e9a47d76b13026 refs/tags/1.0.0\na935cfe9445cc5218fbdd7e0afb35aa1587fff61 refs/tags/1.0.1\n4b83863a9181f054bb695b11b5d663406dfd85d2 refs/tags/1.0.2\n295145fddaa4fe29c1e71a5044d968a84f9dbf69 refs/tags/1.1.0\n8f74ea4e5718436c47305b4488842e6458a13dac refs/tags/1.1.1\n4135bb291e53d615a976e997c44fb2bd9e1ad343 refs/tags/1.1.10\n8c09dbcd16612f5989065db02ea2e7a752dd2656 refs/tags/1.1.11\naedfebdb6c016431d84b0c07cf181b957a900640 refs/tags/1.1.12\n2c2e958366ef6998115740bdf110588d730e5738 refs/tags/1.1.2\nbecc77258321e6ec40d89efdddf37bafd0d07fc3 refs/tags/1.1.3\ne070d7c9853bf94c35b81cf0c0a8980c2449bb22 refs/tags/1.1.4\n12c986cbbf65e8571a486e9230808bf887e5f04f refs/tags/1.1.5\n63df8986f5b56913b02d26954fa033eeaf43714c refs/tags/1.1.6\n38e02c9c6bd728b043036fe0d1894d774cab3108 refs/tags/1.1.7\n3c3879fff16450d28ade79a6b08982bf5cefc061 refs/tags/1.1.8\ne32b811b3b2e70a1d189d7a663bc2583e9c18f96 refs/tags/1.1.9\n0c1b4277c08197ce7e7e0aa2bad91d909fcd96ac refs/tags/2.0.0\n"),
+    toData("testes", GitCurrentCommit, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitMergeBase, 0, "d9db2ad09aa38fc26625341e1b666602959e144f\n416eec87a5ae39a1a6035552e9e9a47d76b13026\nd9db2ad09aa38fc26625341e1b666602959e144f\n"),
+    toData("testes", GitCheckout, 0, "416eec87a5ae39a1a6035552e9e9a47d76b13026"),
+
+    toData("grok", GitDiff, 0, ""),
+    toData("grok", GitTags, 0, "2ca193c31fa2377c1e991a080d60ca3215ff6cf0 refs/tags/0.0.1\n48007554b21ba2f65c726ae2fdda88d621865b4a refs/tags/0.0.2\n7092a0286421c7818cd335cca9ebc72d03d866c2 refs/tags/0.0.3\n62707b8ac684efac35d301dbde57dc750880268e refs/tags/0.0.4\n876f2504e0c2f785ffd2cf65a78e2aea474fa8aa refs/tags/0.0.5\nb7eb1f2501aa2382cb3a38353664a13af62a9888 refs/tags/0.0.6\nf5d818bfd6038884b3d8b531c58484ded20a58a4 refs/tags/0.1.0\n961eaddea49c3144d130d105195583d3f11fb6c6 refs/tags/0.2.0\n15ab8ed8d4f896232a976a9008548bd53af72a66 refs/tags/0.2.1\n426a7d7d4603f77ced658e73ad7f3f582413f6cd refs/tags/0.3.0\n83cf7a39b2fe897786fb0fe01a7a5933c3add286 refs/tags/0.3.1\n8d2e3c900edbc95fa0c036fd76f8e4f814aef2c1 refs/tags/0.3.2\n48b43372f49a3bb4dc0969d82a0fca183fb94662 refs/tags/0.3.3\n9ca947a3009ea6ba17814b20eb953272064eb2e6 refs/tags/0.4.0\n1b5643d04fba6d996a16d1ffc13d034a40003f8f refs/tags/0.5.0\n486b0eb580b1c465453d264ac758cc490c19c33e refs/tags/0.5.1\naedb0d9497390e20b9d2541cef2bb05a5cda7a71 refs/tags/0.5.2\n"),
+    toData("grok", GitCurrentCommit, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n"),
+    toData("grok", GitMergeBase, 0, "4e6526a91a23eaec778184e16ce9a34d25d48bdc\n62707b8ac684efac35d301dbde57dc750880268e\n349c15fd1e03f1fcdd81a1edefba3fa6116ab911\n"),
+    toData("grok", GitCheckout, 0, "62707b8ac684efac35d301dbde57dc750880268e"),
+
+    toData("nim-bytes2human", GitDiff, 0, ""),
+    toData("nim-bytes2human", GitTags, 0, ""),
+    toData("nim-bytes2human", GitCurrentcommit, 0, "ec2c1a758cabdd4751a06c8ebf2b923f19e32731\n")
+  ]
+
+#[
+Current directory is now E:\atlastest\nim-bytes2human
+cmd git diff args [] --> ("", 0)
+cmd git show-ref --tags args [] --> ("", 1)
+cmd git log -n 1 --format=%H args [] --> (, 0)
+[Warning] (nim-bytes2human) package has no tagged releases
+nimble E:\atlastest\nim-bytes2human\bytes2human.nimble info (requires: @["nim >= 1.0.0"], srcDir: "src", tasks: @[])
+[Error] There were problems.
+Error: execution of an external program failed: 'E:\nim\tools\atlas\atlas.exe clone https://github.com/disruptek/balls'
+]#
diff --git a/atlas/tests/balls.nimble b/atlas/tests/balls.nimble
new file mode 100644
index 000000000..143e757e9
--- /dev/null
+++ b/atlas/tests/balls.nimble
@@ -0,0 +1,32 @@
+version = "3.4.1"
+author = "disruptek"
+description = "a unittest framework with balls 🔴🟡🟢"
+license = "MIT"
+
+# requires newTreeFrom
+requires "https://github.com/disruptek/grok >= 0.5.0 & < 1.0.0"
+requires "https://github.com/disruptek/ups < 1.0.0"
+requires "https://github.com/planetis-m/sync#810bd2d"
+#requires "https://github.com/c-blake/cligen < 2.0.0"
+
+bin = @["balls"]            # build the binary for basic test running
+installExt = @["nim"]       # we need to install balls.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+#installFiles = @["balls.nim"] # https://github.com/nim-lang/Nim/issues/16661
+
+task test, "run tests for ci":
+  when defined(windows):
+    exec "balls.cmd"
+  else:
+    exec "balls"
+
+task demo, "produce a demo":
+  exec "nim c --define:release balls.nim"
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/fails.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger -f --out=\$1 tests/test.nim""""
+  exec "nim c --define:release --define:ballsDry balls.nim"
+  exec """demo docs/runner.svg "balls""""
+
diff --git a/atlas/tests/grok.nimble b/atlas/tests/grok.nimble
new file mode 100644
index 000000000..1b6d77c08
--- /dev/null
+++ b/atlas/tests/grok.nimble
@@ -0,0 +1,5 @@
+version = "0.0.4"
+author = "disruptek"
+description = "don't read too much into it"
+license = "MIT"
+requires "nim >= 1.0.0"
diff --git a/atlas/tests/nim-bytes2human.nimble b/atlas/tests/nim-bytes2human.nimble
new file mode 100644
index 000000000..9f3ae2479
--- /dev/null
+++ b/atlas/tests/nim-bytes2human.nimble
@@ -0,0 +1,7 @@
+version     = "0.2.2"
+author      = "Juan Carlos"
+description = "Convert bytes to kilobytes, megabytes, gigabytes, etc."
+license     = "MIT"
+srcDir      = "src"
+
+requires "nim >= 1.0.0"  # https://github.com/juancarlospaco/nim-bytes2human/issues/2#issue-714338524
diff --git a/atlas/tests/nim.cfg b/atlas/tests/nim.cfg
new file mode 100644
index 000000000..3982b12bb
--- /dev/null
+++ b/atlas/tests/nim.cfg
@@ -0,0 +1,10 @@
+############# begin Atlas config section ##########
+--noNimblePath
+--path:"../balls"
+--path:"../grok"
+--path:"../ups"
+--path:"../sync"
+--path:"../npeg/src"
+--path:"../testes"
+--path:"../nim-bytes2human/src"
+############# end Atlas config section   ##########
diff --git a/atlas/tests/npeg.nimble b/atlas/tests/npeg.nimble
new file mode 100644
index 000000000..e71fc5aa5
--- /dev/null
+++ b/atlas/tests/npeg.nimble
@@ -0,0 +1,48 @@
+# Package
+
+version       = "0.24.1"
+author        = "Ico Doornekamp"
+description   = "a PEG library"
+license       = "MIT"
+srcDir        = "src"
+installExt    = @["nim"]
+
+# Dependencies
+
+requires "nim >= 0.19.0"
+
+# Test
+
+task test, "Runs the test suite":
+  exec "nimble testc && nimble testcpp && nimble testarc && nimble testjs"
+
+task testc, "C tests":
+  exec "nim c -r tests/tests.nim"
+
+task testcpp, "CPP tests":
+  exec "nim cpp -r tests/tests.nim"
+
+task testjs, "JS tests":
+  exec "nim js -r tests/tests.nim"
+
+task testdanger, "Runs the test suite in danger mode":
+  exec "nim c -d:danger -r tests/tests.nim"
+
+task testwin, "Mingw tests":
+  exec "nim c -d:mingw tests/tests.nim && wine tests/tests.exe"
+
+task test32, "32 bit tests":
+  exec "nim c --cpu:i386 --passC:-m32 --passL:-m32 tests/tests.nim && tests/tests"
+
+task testall, "Test all":
+  exec "nimble test && nimble testcpp && nimble testdanger && nimble testjs && nimble testwin"
+
+when (NimMajor, NimMinor) >= (1, 1):
+  task testarc, "--gc:arc tests":
+    exec "nim c --gc:arc -r tests/tests.nim"
+else:
+  task testarc, "--gc:arc tests":
+    exec "true"
+
+task perf, "Test performance":
+  exec "nim cpp -r -d:danger tests/performance.nim"
diff --git a/atlas/tests/packages/packages.json b/atlas/tests/packages/packages.json
new file mode 100644
index 000000000..d054a201b
--- /dev/null
+++ b/atlas/tests/packages/packages.json
@@ -0,0 +1,36 @@
+[
+  {
+    "name": "bytes2human",
+    "url": "https://github.com/juancarlospaco/nim-bytes2human",
+    "method": "git",
+    "tags": [
+      "bytes",
+      "human",
+      "minimalism",
+      "size"
+    ],
+    "description": "Convert bytes to kilobytes, megabytes, gigabytes, etc.",
+    "license": "LGPLv3",
+    "web": "https://github.com/juancarlospaco/nim-bytes2human"
+  },
+  {
+    "name": "npeg",
+    "url": "https://github.com/zevv/npeg",
+    "method": "git",
+    "tags": [
+      "PEG",
+      "parser",
+      "parsing",
+      "regexp",
+      "regular",
+      "grammar",
+      "lexer",
+      "lexing",
+      "pattern",
+      "matching"
+    ],
+    "description": "PEG (Parsing Expression Grammars) string matching library for Nim",
+    "license": "MIT",
+    "web": "https://github.com/zevv/npeg"
+  }
+]
diff --git a/atlas/tests/sync.nimble b/atlas/tests/sync.nimble
new file mode 100644
index 000000000..a07ae8925
--- /dev/null
+++ b/atlas/tests/sync.nimble
@@ -0,0 +1,10 @@
+# Package
+
+version     = "1.4.0"
+author      = "Antonis Geralis"
+description = "Useful synchronization primitives."
+license     = "MIT"
+
+# Deps
+
+requires "nim >= 1.0.0"
diff --git a/atlas/tests/testes.nimble b/atlas/tests/testes.nimble
new file mode 100644
index 000000000..60fe1d508
--- /dev/null
+++ b/atlas/tests/testes.nimble
@@ -0,0 +1,23 @@
+version = "1.0.0"
+author = "disruptek"
+description = "a cure for salty testes"
+license = "MIT"
+
+#requires "cligen >= 0.9.41 & <= 0.9.45"
+#requires "bump >= 1.8.18 & < 2.0.0"
+requires "https://github.com/disruptek/grok >= 0.0.4 & < 1.0.0"
+requires "https://github.com/juancarlospaco/nim-bytes2human"
+
+bin = @["testes"]           # build the binary for basic test running
+installExt = @["nim"]       # we need to install testes.nim also
+skipDirs = @["tests"]       # so stupid...  who doesn't want tests?
+
+task test, "run tests for ci":
+  exec "nim c --run testes.nim"
+
+task demo, "produce a demo":
+  when (NimMajor, NimMinor) != (1, 0):
+    echo "due to nim bug #16307, use nim-1.0"
+    quit 1
+  exec """demo docs/demo.svg "nim c --out=\$1 examples/balls.nim""""
+  exec """demo docs/clean.svg "nim c --define:danger --out=\$1 tests/testicles.nim""""
diff --git a/atlas/tests/ups.nimble b/atlas/tests/ups.nimble
new file mode 100644
index 000000000..d91abbe60
--- /dev/null
+++ b/atlas/tests/ups.nimble
@@ -0,0 +1,13 @@
+version = "0.0.2"
+author = "disruptek"
+description = "a package handler"
+license = "MIT"
+
+requires "npeg >= 0.23.2 & < 1.0.0"
+requires "https://github.com/disruptek/testes >= 1.0.0 & < 2.0.0"
+
+task test, "run tests":
+  when defined(windows):
+    exec "testes.cmd"
+  else:
+    exec findExe"testes"
diff --git a/atlas/tests/ws_conflict/atlas.workspace b/atlas/tests/ws_conflict/atlas.workspace
new file mode 100644
index 000000000..21954fea1
--- /dev/null
+++ b/atlas/tests/ws_conflict/atlas.workspace
@@ -0,0 +1,2 @@
+deps=""
+overrides="url.rules"
diff --git a/atlas/tests/ws_conflict/source/apkg/apkg.nimble b/atlas/tests/ws_conflict/source/apkg/apkg.nimble
new file mode 100644
index 000000000..11065ec15
--- /dev/null
+++ b/atlas/tests/ws_conflict/source/apkg/apkg.nimble
@@ -0,0 +1,4 @@
+# require first b and then c
+
+requires "https://github.com/bpkg >= 1.0"
+requires "https://github.com/cpkg >= 1.0"
diff --git a/atlas/tests/ws_conflict/source/bpkg@1.0/bpkg.nimble b/atlas/tests/ws_conflict/source/bpkg@1.0/bpkg.nimble
new file mode 100644
index 000000000..f70ad45cb
--- /dev/null
+++ b/atlas/tests/ws_conflict/source/bpkg@1.0/bpkg.nimble
@@ -0,0 +1 @@
+requires "https://github.com/cpkg >= 2.0"
diff --git a/atlas/tests/ws_conflict/source/cpkg@1.0/cpkg.nimble b/atlas/tests/ws_conflict/source/cpkg@1.0/cpkg.nimble
new file mode 100644
index 000000000..0fbd587aa
--- /dev/null
+++ b/atlas/tests/ws_conflict/source/cpkg@1.0/cpkg.nimble
@@ -0,0 +1 @@
+# No dependency here!
diff --git a/atlas/tests/ws_conflict/source/cpkg@2.0/cpkg.nimble b/atlas/tests/ws_conflict/source/cpkg@2.0/cpkg.nimble
new file mode 100644
index 000000000..381fae111
--- /dev/null
+++ b/atlas/tests/ws_conflict/source/cpkg@2.0/cpkg.nimble
@@ -0,0 +1 @@
+requires "https://github.com/dpkg >= 1.0"
diff --git a/atlas/tests/ws_conflict/source/dpkg/dpkg.nimble b/atlas/tests/ws_conflict/source/dpkg/dpkg.nimble
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/atlas/tests/ws_conflict/source/dpkg/dpkg.nimble
diff --git a/atlas/tests/ws_conflict/url.rules b/atlas/tests/ws_conflict/url.rules
new file mode 100644
index 000000000..c641cf990
--- /dev/null
+++ b/atlas/tests/ws_conflict/url.rules
@@ -0,0 +1 @@
+https://github.com/$+ -> file://./source/$#
generated by cgit-pink 1.4.1-2-gfad0 (git 2.36.2.497.gbbea4dcf42) at 2025-02-04 22:40:49 +0000