Atlas: --project switch and better graph representation (#21971)

1 files changed, 1166 insertions, 0 deletions

diff --git a/atlas/atlas.nim b/atlas/atlas.nim
new file mode 100644
index 000000000..9793f2637
--- /dev/null
+++ b/atlas/atlas.nim
@@ -0,0 +1,1166 @@
+#
+#           Atlas Package Cloner
+#        (c) Copyright 2021 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Simple tool to automate frequent workflows: Can "clone"
+## a Nimble dependency and its dependencies recursively.
+
+import std / [parseopt, strutils, os, osproc, tables, sets, json, jsonutils,
+  parsecfg, streams, terminal, strscans, hashes]
+import parse_requires, osutils, packagesjson, compiledpatterns
+
+from unicode import nil
+
+const
+  Version = "0.4"
+  LockFileName = "atlas.lock"
+  AtlasWorkspace = "atlas.workspace"
+  Usage = "atlas - Nim Package Cloner Version " & Version & """
+
+  (c) 2021 Andreas Rumpf
+Usage:
+  atlas [options] [command] [arguments]
+Command:
+  init                  initializes the current directory as a workspace
+    --deps=DIR          use DIR as the directory for dependencies
+                        (default: store directly in the workspace)
+
+  use url|pkgname       clone a package and all of its dependencies and make
+                        it importable for the current project
+  clone url|pkgname     clone a package and all of its dependencies
+  update url|pkgname    update a package and all of its dependencies
+  install proj.nimble   use the .nimble file to setup the project's dependencies
+  search keyw keywB...  search for package that contains the given keywords
+  extract file.nimble   extract the requirements and custom commands from
+                        the given Nimble file
+  updateProjects [filter]
+                        update every project that has a remote
+                        URL that matches `filter` if a filter is given
+  updateDeps [filter]
+                        update every dependency that has a remote
+                        URL that matches `filter` if a filter is given
+  tag [major|minor|patch]
+                        add and push a new tag, input must be one of:
+                        ['major'|'minor'|'patch'] or a SemVer tag like ['1.0.3']
+                        or a letter ['a'..'z']: a.b.c.d.e.f.g
+  build|test|doc|tasks  currently delegates to `nimble build|test|doc`
+  task <taskname>       currently delegates to `nimble <taskname>`
+  env <nimversion>      setup a Nim virtual environment
+
+Options:
+  --keepCommits         do not perform any `git checkouts`
+  --cfgHere             also create/maintain a nim.cfg in the current
+                        working directory
+  --workspace=DIR       use DIR as workspace
+  --project=DIR         use DIR as the current project
+  --genlock             generate a lock file (use with `clone` and `update`)
+  --uselock             use the lock file for the build
+  --autoinit            auto initialize a workspace
+  --colors=on|off       turn on|off colored output
+  --showGraph           show the dependency graph
+  --version             show the version
+  --help                show this help
+"""
+
+proc writeHelp() =
+  stdout.write(Usage)
+  stdout.flushFile()
+  quit(0)
+
+proc writeVersion() =
+  stdout.write(Version & "\n")
+  stdout.flushFile()
+  quit(0)
+
+const
+  MockupRun = defined(atlasTests)
+  TestsDir = "atlas/tests"
+
+type
+  LockOption = enum
+    noLock, genLock, useLock
+
+  LockFileEntry = object
+    dir, url, commit: string
+
+  PackageName = distinct string
+  CfgPath = distinct string # put into a config `--path:"../x"`
+  DepRelation = enum
+    normal, strictlyLess, strictlyGreater
+
+  SemVerField = enum
+    major, minor, patch
+
+  Dependency = object
+    name: PackageName
+    url, commit: string
+    rel: DepRelation # "requires x < 1.0" is silly, but Nimble allows it so we have too.
+    self: int # position in the graph
+    parents: seq[int] # why we need this dependency
+    active: bool
+  DepGraph = object
+    nodes: seq[Dependency]
+    processed: Table[string, int] # the key is (url / commit)
+    byName: Table[PackageName, seq[int]]
+
+  AtlasContext = object
+    projectDir, workspace, depsDir, currentDir: string
+    hasPackageList: bool
+    keepCommits: bool
+    cfgHere: bool
+    usesOverrides: bool
+    p: Table[string, string] # name -> url mapping
+    errors, warnings: int
+    overrides: Patterns
+    lockOption: LockOption
+    lockFileToWrite: seq[LockFileEntry]
+    lockFileToUse: Table[string, LockFileEntry]
+    when MockupRun:
+      step: int
+      mockupSuccess: bool
+    noColors: bool
+    showGraph: bool
+
+proc `==`*(a, b: CfgPath): bool {.borrow.}
+
+proc `==`*(a, b: PackageName): bool {.borrow.}
+proc hash*(a: PackageName): Hash {.borrow.}
+
+const
+  InvalidCommit = "<invalid commit>"
+  ProduceTest = false
+
+type
+  Command = enum
+    GitDiff = "git diff",
+    GitTag = "git tag",
+    GitTags = "git show-ref --tags",
+    GitLastTaggedRef = "git rev-list --tags --max-count=1",
+    GitDescribe = "git describe",
+    GitRevParse = "git rev-parse",
+    GitCheckout = "git checkout",
+    GitPush = "git push origin",
+    GitPull = "git pull",
+    GitCurrentCommit = "git log -n 1 --format=%H"
+    GitMergeBase = "git merge-base"
+
+include testdata
+
+proc silentExec(cmd: string; args: openArray[string]): (string, int) =
+  var cmdLine = cmd
+  for i in 0..<args.len:
+    cmdLine.add ' '
+    cmdLine.add quoteShell(args[i])
+  result = osproc.execCmdEx(cmdLine)
+
+proc nimbleExec(cmd: string; args: openArray[string]) =
+  var cmdLine = "nimble " & cmd
+  for i in 0..<args.len:
+    cmdLine.add ' '
+    cmdLine.add quoteShell(args[i])
+  discard os.execShellCmd(cmdLine)
+
+proc exec(c: var AtlasContext; cmd: Command; args: openArray[string]): (string, int) =
+  when MockupRun:
+    assert TestLog[c.step].cmd == cmd, $(TestLog[c.step].cmd, cmd)
+    case cmd
+    of GitDiff, GitTag, GitTags, GitLastTaggedRef, GitDescribe, GitRevParse, GitPush, GitPull, GitCurrentCommit:
+      result = (TestLog[c.step].output, TestLog[c.step].exitCode)
+    of GitCheckout:
+      assert args[0] == TestLog[c.step].output
+    of GitMergeBase:
+      let tmp = TestLog[c.step].output.splitLines()
+      assert tmp.len == 4, $tmp.len
+      assert tmp[0] == args[0]
+      assert tmp[1] == args[1]
+      assert tmp[3] == ""
+      result[0] = tmp[2]
+      result[1] = TestLog[c.step].exitCode
+    inc c.step
+  else:
+    result = silentExec($cmd, args)
+    when ProduceTest:
+      echo "cmd ", cmd, " args ", args, " --> ", result
+
+proc cloneUrl(c: var AtlasContext; url, dest: string; cloneUsingHttps: bool): string =
+  when MockupRun:
+    result = ""
+  else:
+    result = osutils.cloneUrl(url, dest, cloneUsingHttps)
+    when ProduceTest:
+      echo "cloned ", url, " into ", dest
+
+template withDir*(c: var AtlasContext; dir: string; body: untyped) =
+  when MockupRun:
+    body
+  else:
+    let oldDir = getCurrentDir()
+    try:
+      when ProduceTest:
+        echo "Current directory is now ", dir
+      setCurrentDir(dir)
+      body
+    finally:
+      setCurrentDir(oldDir)
+
+proc extractRequiresInfo(c: var AtlasContext; nimbleFile: string): NimbleFileInfo =
+  result = extractRequiresInfo(nimbleFile)
+  when ProduceTest:
+    echo "nimble ", nimbleFile, " info ", result
+
+proc toDepRelation(s: string): DepRelation =
+  case s
+  of "<": strictlyLess
+  of ">": strictlyGreater
+  else: normal
+
+proc isCleanGit(c: var AtlasContext): string =
+  result = ""
+  let (outp, status) = exec(c, GitDiff, [])
+  if outp.len != 0:
+    result = "'git diff' not empty"
+  elif status != 0:
+    result = "'git diff' returned non-zero"
+
+proc message(c: var AtlasContext; category: string; p: PackageName; arg: string) =
+  var msg = category & "(" & p.string & ") " & arg
+  stdout.writeLine msg
+
+proc warn(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Warning] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgYellow, styleBright, "[Warning] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+  inc c.warnings
+
+proc error(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Error] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgRed, styleBright, "[Error] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+  inc c.errors
+
+proc info(c: var AtlasContext; p: PackageName; arg: string) =
+  if c.noColors:
+    message(c, "[Info] ", p, arg)
+  else:
+    stdout.styledWriteLine(fgGreen, styleBright, "[Info] ", resetStyle, fgCyan, "(", p.string, ")", resetStyle, " ", arg)
+
+template projectFromCurrentDir(): PackageName = PackageName(c.currentDir.splitPath.tail)
+
+proc readableFile(s: string): string = relativePath(s, getCurrentDir())
+
+proc sameVersionAs(tag, ver: string): bool =
+  const VersionChars = {'0'..'9', '.'}
+
+  proc safeCharAt(s: string; i: int): char {.inline.} =
+    if i >= 0 and i < s.len: s[i] else: '\0'
+
+  let idx = find(tag, ver)
+  if idx >= 0:
+    # we found the version as a substring inside the `tag`. But we
+    # need to watch out the the boundaries are not part of a
+    # larger/different version number:
+    result = safeCharAt(tag, idx-1) notin VersionChars and
+      safeCharAt(tag, idx+ver.len) notin VersionChars
+
+proc gitDescribeRefTag(c: var AtlasContext; commit: string): string =
+  let (lt, status) = exec(c, GitDescribe, ["--tags", commit])
+  result = if status == 0: strutils.strip(lt) else: ""
+
+proc getLastTaggedCommit(c: var AtlasContext): string =
+  let (ltr, status) = exec(c, GitLastTaggedRef, [])
+  if status == 0:
+    let lastTaggedRef = ltr.strip()
+    let lastTag = gitDescribeRefTag(c, lastTaggedRef)
+    if lastTag.len != 0:
+      result = lastTag
+
+proc versionToCommit(c: var AtlasContext; d: Dependency): string =
+  let (outp, status) = exec(c, GitTags, [])
+  if status == 0:
+    var useNextOne = false
+    for line in splitLines(outp):
+      let commitsAndTags = strutils.splitWhitespace(line)
+      if commitsAndTags.len == 2:
+        case d.rel
+        of normal:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyLess:
+          if d.commit == InvalidCommit:
+            return getLastTaggedCommit(c)
+          elif not commitsAndTags[1].sameVersionAs(d.commit):
+            return commitsAndTags[0]
+        of strictlyGreater:
+          if commitsAndTags[1].sameVersionAs(d.commit):
+            useNextOne = true
+          elif useNextOne:
+            return commitsAndTags[0]
+
+  return ""
+
+proc shortToCommit(c: var AtlasContext; short: string): string =
+  let (cc, status) = exec(c, GitRevParse, [short])
+  result = if status == 0: strutils.strip(cc) else: ""
+
+proc checkoutGitCommit(c: var AtlasContext; p: PackageName; commit: string) =
+  let (_, status) = exec(c, GitCheckout, [commit])
+  if status != 0:
+    error(c, p, "could not checkout commit " & commit)
+
+proc gitPull(c: var AtlasContext; p: PackageName) =
+  let (_, status) = exec(c, GitPull, [])
+  if status != 0:
+    error(c, p, "could not 'git pull'")
+
+proc gitTag(c: var AtlasContext; tag: string) =
+  let (_, status) = exec(c, GitTag, [tag])
+  if status != 0:
+    error(c, c.projectDir.PackageName, "could not 'git tag " & tag & "'")
+
+proc pushTag(c: var AtlasContext; tag: string) =
+  let (outp, status) = exec(c, GitPush, [tag])
+  if status != 0:
+    error(c, c.projectDir.PackageName, "could not 'git push " & tag & "'")
+  elif outp.strip() == "Everything up-to-date":
+    info(c, c.projectDir.PackageName, "is up-to-date")
+  else:
+    info(c, c.projectDir.PackageName, "successfully pushed tag: " & tag)
+
+proc incrementTag(c: var AtlasContext; lastTag: string; field: Natural): string =
+  var startPos =
+    if lastTag[0] in {'0'..'9'}: 0
+    else: 1
+  var endPos = lastTag.find('.', startPos)
+  if field >= 1:
+    for i in 1 .. field:
+      if endPos == -1:
+        error c, projectFromCurrentDir(), "the last tag '" & lastTag & "' is missing . periods"
+        return ""
+      startPos = endPos + 1
+      endPos = lastTag.find('.', startPos)
+  if endPos == -1:
+    endPos = len(lastTag)
+  let patchNumber = parseInt(lastTag[startPos..<endPos])
+  lastTag[0..<startPos] & $(patchNumber + 1) & lastTag[endPos..^1]
+
+proc incrementLastTag(c: var AtlasContext; field: Natural): string =
+  let (ltr, status) = exec(c, GitLastTaggedRef, [])
+  if status == 0:
+    let
+      lastTaggedRef = ltr.strip()
+      lastTag = gitDescribeRefTag(c, lastTaggedRef)
+      currentCommit = exec(c, GitCurrentCommit, [])[0].strip()
+
+    if lastTaggedRef == currentCommit:
+      info c, c.projectDir.PackageName, "the current commit '" & currentCommit & "' is already tagged '" & lastTag & "'"
+      lastTag
+    else:
+      incrementTag(c, lastTag, field)
+  else: "v0.0.1" # assuming no tags have been made yet
+
+proc tag(c: var AtlasContext; tag: string) =
+  gitTag(c, tag)
+  pushTag(c, tag)
+
+proc tag(c: var AtlasContext; field: Natural) =
+  let oldErrors = c.errors
+  let newTag = incrementLastTag(c, field)
+  if c.errors == oldErrors:
+    tag(c, newTag)
+
+proc updatePackages(c: var AtlasContext) =
+  if dirExists(c.workspace / PackagesDir):
+    withDir(c, c.workspace / PackagesDir):
+      gitPull(c, PackageName PackagesDir)
+  else:
+    withDir c, c.workspace:
+      let err = cloneUrl(c, "https://github.com/nim-lang/packages", PackagesDir, false)
+      if err != "":
+        error c, PackageName(PackagesDir), err
+
+proc fillPackageLookupTable(c: var AtlasContext) =
+  if not c.hasPackageList:
+    c.hasPackageList = true
+    when not MockupRun:
+      updatePackages(c)
+    let plist = getPackages(when MockupRun: TestsDir else: c.workspace)
+    for entry in plist:
+      c.p[unicode.toLower entry.name] = entry.url
+
+proc toUrl(c: var AtlasContext; p: string): string =
+  if p.isUrl:
+    if c.usesOverrides:
+      result = c.overrides.substitute(p)
+      if result.len > 0: return result
+    result = p
+  else:
+    # either the project name or the URL can be overwritten!
+    if c.usesOverrides:
+      result = c.overrides.substitute(p)
+      if result.len > 0: return result
+
+    fillPackageLookupTable(c)
+    result = c.p.getOrDefault(unicode.toLower p)
+
+    if result.len == 0:
+      result = getUrlFromGithub(p)
+      if result.len == 0:
+        inc c.errors
+
+    if c.usesOverrides:
+      let newUrl = c.overrides.substitute(result)
+      if newUrl.len > 0: return newUrl
+
+proc toName(p: string): PackageName =
+  if p.isUrl:
+    result = PackageName splitFile(p).name
+  else:
+    result = PackageName p
+
+proc generateDepGraph(c: var AtlasContext; g: DepGraph) =
+  proc repr(w: Dependency): string = w.url / w.commit
+
+  var dotGraph = ""
+  for i in 0 ..< g.nodes.len:
+    dotGraph.addf("\"$1\" [label=\"$2\"];\n", [g.nodes[i].repr, if g.nodes[i].active: "" else: "unused"])
+  for i in 0 ..< g.nodes.len:
+    for p in items g.nodes[i].parents:
+      if p >= 0:
+        dotGraph.addf("\"$1\" -> \"$2\";\n", [g.nodes[p].repr, g.nodes[i].repr])
+  let dotFile = c.workspace / "deps.dot"
+  writeFile(dotFile, "digraph deps {\n$1}\n" % dotGraph)
+  let graphvizDotPath = findExe("dot")
+  if graphvizDotPath.len == 0:
+    #echo("gendepend: Graphviz's tool dot is required, " &
+    #  "see https://graphviz.org/download for downloading")
+    discard
+  else:
+    discard execShellCmd("dot -Tpng -odeps.png " & quoteShell(dotFile))
+
+proc showGraph(c: var AtlasContext; g: DepGraph) =
+  if c.showGraph:
+    generateDepGraph c, g
+
+proc needsCommitLookup(commit: string): bool {.inline.} =
+  '.' in commit or commit == InvalidCommit
+
+proc isShortCommitHash(commit: string): bool {.inline.} =
+  commit.len >= 4 and commit.len < 40
+
+proc getRequiredCommit(c: var AtlasContext; w: Dependency): string =
+  if needsCommitLookup(w.commit): versionToCommit(c, w)
+  elif isShortCommitHash(w.commit): shortToCommit(c, w.commit)
+  else: w.commit
+
+proc getRemoteUrl(): string =
+  execProcess("git config --get remote.origin.url").strip()
+
+proc genLockEntry(c: var AtlasContext; w: Dependency; dir: string) =
+  let url = getRemoteUrl()
+  var commit = getRequiredCommit(c, w)
+  if commit.len == 0 or needsCommitLookup(commit):
+    commit = execProcess("git log -1 --pretty=format:%H").strip()
+  c.lockFileToWrite.add LockFileEntry(dir: relativePath(dir, c.workspace, '/'), url: url, commit: commit)
+
+proc commitFromLockFile(c: var AtlasContext; dir: string): string =
+  let url = getRemoteUrl()
+  let d = relativePath(dir, c.workspace, '/')
+  if d in c.lockFileToUse:
+    result = c.lockFileToUse[d].commit
+    let wanted = c.lockFileToUse[d].url
+    if wanted != url:
+      error c, PackageName(d), "remote URL has been compromised: got: " &
+          url & " but wanted: " & wanted
+  else:
+    error c, PackageName(d), "package is not listed in the lock file"
+
+proc dependencyDir(c: AtlasContext; w: Dependency): string =
+  result = c.workspace / w.name.string
+  if not dirExists(result):
+    result = c.depsDir / w.name.string
+
+proc selectNode(c: var AtlasContext; g: var DepGraph; w: Dependency) =
+  # all other nodes of the same project name are not active
+  for e in items g.byName[w.name]:
+    g.nodes[e].active = e == w.self
+
+proc checkoutCommit(c: var AtlasContext; g: var DepGraph; w: Dependency) =
+  let dir = dependencyDir(c, w)
+  withDir c, dir:
+    if c.lockOption == genLock:
+      genLockEntry(c, w, dir)
+    elif c.lockOption == useLock:
+      checkoutGitCommit(c, w.name, commitFromLockFile(c, dir))
+    elif w.commit.len == 0 or cmpIgnoreCase(w.commit, "head") == 0:
+      gitPull(c, w.name)
+    else:
+      let err = isCleanGit(c)
+      if err != "":
+        warn c, w.name, err
+      else:
+        let requiredCommit = getRequiredCommit(c, w)
+        let (cc, status) = exec(c, GitCurrentCommit, [])
+        let currentCommit = strutils.strip(cc)
+        if requiredCommit == "" or status != 0:
+          if requiredCommit == "" and w.commit == InvalidCommit:
+            warn c, w.name, "package has no tagged releases"
+          else:
+            warn c, w.name, "cannot find specified version/commit " & w.commit
+        else:
+          if currentCommit != requiredCommit:
+            # checkout the later commit:
+            # git merge-base --is-ancestor <commit> <commit>
+            let (cc, status) = exec(c, GitMergeBase, [currentCommit, requiredCommit])
+            let mergeBase = strutils.strip(cc)
+            if status == 0 and (mergeBase == currentCommit or mergeBase == requiredCommit):
+              # conflict resolution: pick the later commit:
+              if mergeBase == currentCommit:
+                checkoutGitCommit(c, w.name, requiredCommit)
+                selectNode c, g, w
+            else:
+              checkoutGitCommit(c, w.name, requiredCommit)
+              selectNode c, g, w
+              when false:
+                warn c, w.name, "do not know which commit is more recent:",
+                  currentCommit, "(current) or", w.commit, " =", requiredCommit, "(required)"
+
+proc findNimbleFile(c: AtlasContext; dep: Dependency): string =
+  when MockupRun:
+    result = TestsDir / dep.name.string & ".nimble"
+    doAssert fileExists(result), "file does not exist " & result
+  else:
+    let dir = dependencyDir(c, dep)
+    result = dir / (dep.name.string & ".nimble")
+    if not fileExists(result):
+      result = ""
+      for x in walkFiles(dir / "*.nimble"):
+        if result.len == 0:
+          result = x
+        else:
+          # ambiguous .nimble file
+          return ""
+
+proc addUnique[T](s: var seq[T]; elem: sink T) =
+  if not s.contains(elem): s.add elem
+
+proc addUniqueDep(c: var AtlasContext; g: var DepGraph; parent: int;
+                  tokens: seq[string]; lockfile: Table[string, LockFileEntry]) =
+  let pkgName = tokens[0]
+  let oldErrors = c.errors
+  let url = toUrl(c, pkgName)
+  if oldErrors != c.errors:
+    warn c, toName(pkgName), "cannot resolve package name"
+  else:
+    let key = url / tokens[2]
+    if g.processed.hasKey(key):
+      g.nodes[g.processed[key]].parents.addUnique parent
+    else:
+      let self = g.nodes.len
+      g.byName.mgetOrPut(toName(pkgName), @[]).add self
+      g.processed[key] = self
+      if lockfile.contains(pkgName):
+        g.nodes.add Dependency(name: toName(pkgName),
+                            url: lockfile[pkgName].url,
+                            commit: lockfile[pkgName].commit,
+                            rel: normal,
+                            self: self,
+                            parents: @[parent])
+      else:
+        g.nodes.add Dependency(name: toName(pkgName), url: url, commit: tokens[2],
+                               rel: toDepRelation(tokens[1]),
+                               self: self,
+                               parents: @[parent])
+
+template toDestDir(p: PackageName): string = p.string
+
+proc readLockFile(filename: string): Table[string, LockFileEntry] =
+  let jsonAsStr = readFile(filename)
+  let jsonTree = parseJson(jsonAsStr)
+  let data = to(jsonTree, seq[LockFileEntry])
+  result = initTable[string, LockFileEntry]()
+  for d in items(data):
+    result[d.dir] = d
+
+proc collectDeps(c: var AtlasContext; g: var DepGraph; parent: int;
+                 dep: Dependency; nimbleFile: string): CfgPath =
+  # If there is a .nimble file, return the dependency path & srcDir
+  # else return "".
+  assert nimbleFile != ""
+  let nimbleInfo = extractRequiresInfo(c, nimbleFile)
+
+  let lockFilePath = dependencyDir(c, dep) / LockFileName
+  let lockFile = if fileExists(lockFilePath): readLockFile(lockFilePath)
+                 else: initTable[string, LockFileEntry]()
+
+  for r in nimbleInfo.requires:
+    var tokens: seq[string] = @[]
+    for token in tokenizeRequires(r):
+      tokens.add token
+    if tokens.len == 1:
+      # nimx uses dependencies like 'requires "sdl2"'.
+      # Via this hack we map them to the first tagged release.
+      # (See the `isStrictlySmallerThan` logic.)
+      tokens.add "<"
+      tokens.add InvalidCommit
+    elif tokens.len == 2 and tokens[1].startsWith("#"):
+      # Dependencies can also look like 'requires "sdl2#head"
+      var commit = tokens[1][1 .. ^1]
+      tokens[1] = "=="
+      tokens.add commit
+
+    if tokens.len >= 3 and cmpIgnoreCase(tokens[0], "nim") != 0:
+      c.addUniqueDep g, parent, tokens, lockFile
+  result = CfgPath(toDestDir(dep.name) / nimbleInfo.srcDir)
+
+proc collectNewDeps(c: var AtlasContext; g: var DepGraph; parent: int;
+                    dep: Dependency): CfgPath =
+  let nimbleFile = findNimbleFile(c, dep)
+  if nimbleFile != "":
+    result = collectDeps(c, g, parent, dep, nimbleFile)
+  else:
+    result = CfgPath toDestDir(dep.name)
+
+proc selectDir(a, b: string): string = (if dirExists(a): a else: b)
+
+const
+  FileProtocol = "file://"
+  ThisVersion = "current_version.atlas"
+
+proc copyFromDisk(c: var AtlasContext; w: Dependency) =
+  let destDir = toDestDir(w.name)
+  var u = w.url.substr(FileProtocol.len)
+  if u.startsWith("./"): u = c.workspace / u.substr(2)
+  copyDir(selectDir(u & "@" & w.commit, u), destDir)
+  writeFile destDir / ThisVersion, w.commit
+  #echo "WRITTEN ", destDir / ThisVersion
+
+proc isNewerVersion(a, b: string): bool =
+  # only used for testing purposes.
+  if a == InvalidCommit or b == InvalidCommit:
+    return true
+  var amajor, aminor, apatch, bmajor, bminor, bpatch: int
+  if scanf(a, "$i.$i.$i", amajor, aminor, apatch):
+    assert scanf(b, "$i.$i.$i", bmajor, bminor, bpatch)
+    result = (amajor, aminor, apatch) > (bmajor, bminor, bpatch)
+  else:
+    assert scanf(a, "$i.$i", amajor, aminor)
+    assert scanf(b, "$i.$i", bmajor, bminor)
+    result = (amajor, aminor) > (bmajor, bminor)
+
+proc isLaterCommit(destDir, version: string): bool =
+  let oldVersion = try: readFile(destDir / ThisVersion).strip: except: "0.0"
+  result = isNewerVersion(version, oldVersion)
+
+proc traverseLoop(c: var AtlasContext; g: var DepGraph; startIsDep: bool): seq[CfgPath] =
+  result = @[]
+  var i = 0
+  while i < g.nodes.len:
+    let w = g.nodes[i]
+    let destDir = toDestDir(w.name)
+    let oldErrors = c.errors
+
+    if not dirExists(c.workspace / destDir) and not dirExists(c.depsDir / destDir):
+      withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
+        if w.url.startsWith(FileProtocol):
+          copyFromDisk c, w
+        else:
+          let err = cloneUrl(c, w.url, destDir, false)
+          if err != "":
+            error c, w.name, err
+    # assume this is the selected version, it might get overwritten later:
+    selectNode c, g, w
+    if oldErrors == c.errors:
+      if not c.keepCommits:
+        if not w.url.startsWith(FileProtocol):
+          checkoutCommit(c, g, w)
+        else:
+          withDir c, (if i != 0 or startIsDep: c.depsDir else: c.workspace):
+            if isLaterCommit(destDir, w.commit):
+              copyFromDisk c, w
+              selectNode c, g, w
+      # even if the checkout fails, we can make use of the somewhat
+      # outdated .nimble file to clone more of the most likely still relevant
+      # dependencies:
+      result.addUnique collectNewDeps(c, g, i, w)
+    inc i
+
+proc traverse(c: var AtlasContext; start: string; startIsDep: bool): seq[CfgPath] =
+  # returns the list of paths for the nim.cfg file.
+  let url = toUrl(c, start)
+  var g = DepGraph(nodes: @[Dependency(name: toName(start), url: url, commit: "", self: 0)])
+  g.byName.mgetOrPut(toName(start), @[]).add 0
+
+  if url == "":
+    error c, toName(start), "cannot resolve package name"
+    return
+
+  c.projectDir = c.workspace / toDestDir(g.nodes[0].name)
+  if c.lockOption == useLock:
+    c.lockFileToUse = readLockFile(c.projectDir / LockFileName)
+  result = traverseLoop(c, g, startIsDep)
+  if c.lockOption == genLock:
+    writeFile c.projectDir / LockFileName, toJson(c.lockFileToWrite).pretty
+  showGraph c, g
+
+const
+  configPatternBegin = "############# begin Atlas config section ##########\n"
+  configPatternEnd =   "############# end Atlas config section   ##########\n"
+
+proc patchNimCfg(c: var AtlasContext; deps: seq[CfgPath]; cfgPath: string) =
+  var paths = "--noNimblePath\n"
+  for d in deps:
+    let pkgname = toDestDir d.string.PackageName
+    let pkgdir = if dirExists(c.workspace / pkgname): c.workspace / pkgname
+                 else: c.depsDir / pkgName
+    let x = relativePath(pkgdir, cfgPath, '/')
+    paths.add "--path:\"" & x & "\"\n"
+  var cfgContent = configPatternBegin & paths & configPatternEnd
+
+  when MockupRun:
+    assert readFile(TestsDir / "nim.cfg") == cfgContent
+    c.mockupSuccess = true
+  else:
+    let cfg = cfgPath / "nim.cfg"
+    assert cfgPath.len > 0
+    if cfgPath.len > 0 and not dirExists(cfgPath):
+      error(c, c.projectDir.PackageName, "could not write the nim.cfg")
+    elif not fileExists(cfg):
+      writeFile(cfg, cfgContent)
+      info(c, projectFromCurrentDir(), "created: " & cfg.readableFile)
+    else:
+      let content = readFile(cfg)
+      let start = content.find(configPatternBegin)
+      if start >= 0:
+        cfgContent = content.substr(0, start-1) & cfgContent
+        let theEnd = content.find(configPatternEnd, start)
+        if theEnd >= 0:
+          cfgContent.add content.substr(theEnd+len(configPatternEnd))
+      else:
+        cfgContent = content & "\n" & cfgContent
+      if cfgContent != content:
+        # do not touch the file if nothing changed
+        # (preserves the file date information):
+        writeFile(cfg, cfgContent)
+        info(c, projectFromCurrentDir(), "updated: " & cfg.readableFile)
+
+proc fatal*(msg: string) =
+  when defined(debug):
+    writeStackTrace()
+  quit "[Error] " & msg
+
+proc findSrcDir(c: var AtlasContext): string =
+  for nimbleFile in walkPattern(c.currentDir / "*.nimble"):
+    let nimbleInfo = extractRequiresInfo(c, nimbleFile)
+    return c.currentDir / nimbleInfo.srcDir
+  return c.currentDir
+
+proc installDependencies(c: var AtlasContext; nimbleFile: string; startIsDep: bool) =
+  # 1. find .nimble file in CWD
+  # 2. install deps from .nimble
+  var g = DepGraph(nodes: @[])
+  let (_, pkgname, _) = splitFile(nimbleFile)
+  let dep = Dependency(name: toName(pkgname), url: "", commit: "", self: 0)
+  discard collectDeps(c, g, -1, dep, nimbleFile)
+  let paths = traverseLoop(c, g, startIsDep)
+  patchNimCfg(c, paths, if c.cfgHere: c.currentDir else: findSrcDir(c))
+  showGraph c, g
+
+proc updateDir(c: var AtlasContext; dir, filter: string) =
+  for kind, file in walkDir(dir):
+    if kind == pcDir and dirExists(file / ".git"):
+      c.withDir file:
+        let pkg = PackageName(file)
+        let (remote, _) = osproc.execCmdEx("git remote -v")
+        if filter.len == 0 or filter in remote:
+          let diff = isCleanGit(c)
+          if diff != "":
+            warn(c, pkg, "has uncommitted changes; skipped")
+          else:
+            let (branch, _) = osproc.execCmdEx("git rev-parse --abbrev-ref HEAD")
+            if branch.strip.len > 0:
+              let (output, exitCode) = osproc.execCmdEx("git pull origin " & branch.strip)
+              if exitCode != 0:
+                error c, pkg, output
+              else:
+                info(c, pkg, "successfully updated")
+            else:
+              error c, pkg, "could not fetch current branch name"
+
+proc patchNimbleFile(c: var AtlasContext; dep: string): string =
+  let thisProject = c.currentDir.splitPath.tail
+  let oldErrors = c.errors
+  let url = toUrl(c, dep)
+  result = ""
+  if oldErrors != c.errors:
+    warn c, toName(dep), "cannot resolve package name"
+  else:
+    for x in walkFiles(c.currentDir / "*.nimble"):
+      if result.len == 0:
+        result = x
+      else:
+        # ambiguous .nimble file
+        warn c, toName(dep), "cannot determine `.nimble` file; there are multiple to choose from"
+        return ""
+    # see if we have this requirement already listed. If so, do nothing:
+    var found = false
+    if result.len > 0:
+      let nimbleInfo = extractRequiresInfo(c, result)
+      for r in nimbleInfo.requires:
+        var tokens: seq[string] = @[]
+        for token in tokenizeRequires(r):
+          tokens.add token
+        if tokens.len > 0:
+          let oldErrors = c.errors
+          let urlB = toUrl(c, tokens[0])
+          if oldErrors != c.errors:
+            warn c, toName(tokens[0]), "cannot resolve package name; found in: " & result
+          if url == urlB:
+            found = true
+            break
+
+    if not found:
+      let line = "requires \"$1\"\n" % dep.escape("", "")
+      if result.len > 0:
+        let oldContent = readFile(result)
+        writeFile result, oldContent & "\n" & line
+        info(c, toName(thisProject), "updated: " & result.readableFile)
+      else:
+        result = c.currentDir / thisProject & ".nimble"
+        writeFile result, line
+        info(c, toName(thisProject), "created: " & result.readableFile)
+    else:
+      info(c, toName(thisProject), "up to date: " & result.readableFile)
+
+proc detectWorkspace(currentDir: string): string =
+  result = currentDir
+  while result.len > 0:
+    if fileExists(result / AtlasWorkspace):
+      return result
+    result = result.parentDir()
+
+proc absoluteDepsDir(workspace, value: string): string =
+  if value == ".":
+    result = workspace
+  elif isAbsolute(value):
+    result = value
+  else:
+    result = workspace / value
+
+proc autoWorkspace(currentDir: string): string =
+  result = currentDir
+  while result.len > 0 and dirExists(result / ".git"):
+    result = result.parentDir()
+
+proc createWorkspaceIn(workspace, depsDir: string) =
+  if not fileExists(workspace / AtlasWorkspace):
+    writeFile workspace / AtlasWorkspace, "deps=\"$#\"" % escape(depsDir, "", "")
+  createDir absoluteDepsDir(workspace, depsDir)
+
+proc parseOverridesFile(c: var AtlasContext; filename: string) =
+  const Separator = " -> "
+  let path = c.workspace / filename
+  var f: File
+  if open(f, path):
+    c.usesOverrides = true
+    try:
+      var lineCount = 1
+      for line in lines(path):
+        let splitPos = line.find(Separator)
+        if splitPos >= 0 and line[0] != '#':
+          let key = line.substr(0, splitPos-1)
+          let val = line.substr(splitPos+len(Separator))
+          if key.len == 0 or val.len == 0:
+            error c, toName(path), "key/value must not be empty"
+          let err = c.overrides.addPattern(key, val)
+          if err.len > 0:
+            error c, toName(path), "(" & $lineCount & "): " & err
+        else:
+          discard "ignore the line"
+        inc lineCount
+    finally:
+      close f
+  else:
+    error c, toName(path), "cannot open: " & path
+
+proc readConfig(c: var AtlasContext) =
+  let configFile = c.workspace / AtlasWorkspace
+  var f = newFileStream(configFile, fmRead)
+  if f == nil:
+    error c, toName(configFile), "cannot open: " & configFile
+    return
+  var p: CfgParser
+  open(p, f, configFile)
+  while true:
+    var e = next(p)
+    case e.kind
+    of cfgEof: break
+    of cfgSectionStart:
+      discard "who cares about sections"
+    of cfgKeyValuePair:
+      case e.key.normalize
+      of "deps":
+        c.depsDir = absoluteDepsDir(c.workspace, e.value)
+      of "overrides":
+        parseOverridesFile(c, e.value)
+      else:
+        warn c, toName(configFile), "ignored unknown setting: " & e.key
+    of cfgOption:
+      discard "who cares about options"
+    of cfgError:
+      error c, toName(configFile), e.msg
+  close(p)
+
+const
+  BatchFile = """
+@echo off
+set PATH="$1";%PATH%
+"""
+  ShellFile = "export PATH=$1:$$PATH\n"
+
+proc setupNimEnv(c: var AtlasContext; nimVersion: string) =
+  template isDevel(nimVersion: string): bool = nimVersion == "devel"
+
+  template exec(c: var AtlasContext; command: string) =
+    let cmd = command # eval once
+    if os.execShellCmd(cmd) != 0:
+      error c, toName("nim-" & nimVersion), "failed: " & cmd
+      return
+
+  let nimDest = "nim-" & nimVersion
+  if dirExists(c.workspace / nimDest):
+    info c, toName(nimDest), "already exists; remove or rename and try again"
+    return
+
+  var major, minor, patch: int
+  if nimVersion != "devel":
+    if not scanf(nimVersion, "$i.$i.$i", major, minor, patch):
+      error c, toName("nim"), "cannot parse version requirement"
+      return
+  let csourcesVersion =
+    if nimVersion.isDevel or (major == 1 and minor >= 9) or major >= 2:
+      # already uses csources_v2
+      "csources_v2"
+    elif major == 0:
+      "csources" # has some chance of working
+    else:
+      "csources_v1"
+  withDir c, c.workspace:
+    if not dirExists(csourcesVersion):
+      exec c, "git clone https://github.com/nim-lang/" & csourcesVersion
+    exec c, "git clone https://github.com/nim-lang/nim " & nimDest
+  withDir c, c.workspace / csourcesVersion:
+    when defined(windows):
+      exec c, "build.bat"
+    else:
+      let makeExe = findExe("make")
+      if makeExe.len == 0:
+        exec c, "sh build.sh"
+      else:
+        exec c, "make"
+  let nimExe0 = ".." / csourcesVersion / "bin" / "nim".addFileExt(ExeExt)
+  withDir c, c.workspace / nimDest:
+    let nimExe = "bin" / "nim".addFileExt(ExeExt)
+    copyFileWithPermissions nimExe0, nimExe
+    let dep = Dependency(name: toName(nimDest), rel: normal, commit: nimVersion, self: 0)
+    if not nimVersion.isDevel:
+      let commit = versionToCommit(c, dep)
+      if commit.len == 0:
+        error c, toName(nimDest), "cannot resolve version to a commit"
+        return
+      checkoutGitCommit(c, dep.name, commit)
+    exec c, nimExe & " c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch"
+    let kochExe = when defined(windows): "koch.exe" else: "./koch"
+    exec c, kochExe & " boot -d:release --skipUserCfg --skipParentCfg --hints:off"
+    exec c, kochExe & " tools --skipUserCfg --skipParentCfg --hints:off"
+    # remove any old atlas binary that we now would end up using:
+    if cmpPaths(getAppDir(), c.workspace / nimDest / "bin") != 0:
+      removeFile "bin" / "atlas".addFileExt(ExeExt)
+    let pathEntry = (c.workspace / nimDest / "bin")
+    when defined(windows):
+      writeFile "activate.bat", BatchFile % pathEntry.replace('/', '\\')
+      info c, toName(nimDest), "RUN\nnim-" & nimVersion & "\\activate.bat"
+    else:
+      writeFile "activate.sh", ShellFile % pathEntry
+      info c, toName(nimDest), "RUN\nsource nim-" & nimVersion & "/activate.sh"
+
+proc main =
+  var action = ""
+  var args: seq[string] = @[]
+  template singleArg() =
+    if args.len != 1:
+      fatal action & " command takes a single package name"
+
+  template noArgs() =
+    if args.len != 0:
+      fatal action & " command takes no arguments"
+
+  template projectCmd() =
+    if c.projectDir == c.workspace or c.projectDir == c.depsDir:
+      fatal action & " command must be executed in a project, not in the workspace"
+
+  var c = AtlasContext(projectDir: getCurrentDir(), currentDir: getCurrentDir(), workspace: "")
+  var autoinit = false
+  for kind, key, val in getopt():
+    case kind
+    of cmdArgument:
+      if action.len == 0:
+        action = key.normalize
+      else:
+        args.add key
+    of cmdLongOption, cmdShortOption:
+      case normalize(key)
+      of "help", "h": writeHelp()
+      of "version", "v": writeVersion()
+      of "keepcommits": c.keepCommits = true
+      of "workspace":
+        if val == ".":
+          c.workspace = getCurrentDir()
+          createWorkspaceIn c.workspace, c.depsDir
+        elif val.len > 0:
+          c.workspace = val
+          createDir(val)
+          createWorkspaceIn c.workspace, c.depsDir
+        else:
+          writeHelp()
+      of "project":
+        if isAbsolute(val):
+          c.currentDir = val
+        else:
+          c.currentDir = getCurrentDir() / val
+      of "deps":
+        if val.len > 0:
+          c.depsDir = val
+        else:
+          writeHelp()
+      of "cfghere": c.cfgHere = true
+      of "autoinit": autoinit = true
+      of "showgraph": c.showGraph = true
+      of "genlock":
+        if c.lockOption != useLock:
+          c.lockOption = genLock
+        else:
+          writeHelp()
+      of "uselock":
+        if c.lockOption != genLock:
+          c.lockOption = useLock
+        else:
+          writeHelp()
+      of "colors":
+        case val.normalize
+        of "off": c.noColors = true
+        of "on": c.noColors = false
+        else: writeHelp()
+      else: writeHelp()
+    of cmdEnd: assert false, "cannot happen"
+
+  if c.workspace.len > 0:
+    if not dirExists(c.workspace): fatal "Workspace directory '" & c.workspace & "' not found."
+  elif action != "init":
+    when MockupRun:
+      c.workspace = autoWorkspace(c.currentDir)
+    else:
+      c.workspace = detectWorkspace(c.currentDir)
+      if c.workspace.len > 0:
+        readConfig c
+        info c, toName(c.workspace.readableFile), "is the current workspace"
+      elif autoinit:
+        c.workspace = autoWorkspace(c.currentDir)
+        createWorkspaceIn c.workspace, c.depsDir
+      elif action notin ["search", "list"]:
+        fatal "No workspace found. Run `atlas init` if you want this current directory to be your workspace."
+
+  when MockupRun:
+    c.depsDir = c.workspace
+
+  case action
+  of "":
+    fatal "No action."
+  of "init":
+    c.workspace = getCurrentDir()
+    createWorkspaceIn c.workspace, c.depsDir
+  of "clone", "update":
+    singleArg()
+    let deps = traverse(c, args[0], startIsDep = false)
+    patchNimCfg c, deps, if c.cfgHere: c.currentDir else: findSrcDir(c)
+    when MockupRun:
+      if not c.mockupSuccess:
+        fatal "There were problems."
+    else:
+      if c.errors > 0:
+        fatal "There were problems."
+  of "use":
+    projectCmd()
+    singleArg()
+    let nimbleFile = patchNimbleFile(c, args[0])
+    if nimbleFile.len > 0:
+      installDependencies(c, nimbleFile, startIsDep = false)
+  of "install":
+    projectCmd()
+    if args.len > 1:
+      fatal "install command takes a single argument"
+    var nimbleFile = ""
+    if args.len == 1:
+      nimbleFile = args[0]
+    else:
+      for x in walkPattern("*.nimble"):
+        nimbleFile = x
+        break
+    if nimbleFile.len == 0:
+      fatal "could not find a .nimble file"
+    else:
+      installDependencies(c, nimbleFile, startIsDep = true)
+  of "refresh":
+    noArgs()
+    updatePackages(c)
+  of "search", "list":
+    if c.workspace.len != 0:
+      updatePackages(c)
+      search getPackages(c.workspace), args
+    else: search @[], args
+  of "updateprojects":
+    updateDir(c, c.workspace, if args.len == 0: "" else: args[0])
+  of "updatedeps":
+    updateDir(c, c.depsDir, if args.len == 0: "" else: args[0])
+  of "extract":
+    singleArg()
+    if fileExists(args[0]):
+      echo toJson(extractRequiresInfo(args[0]))
+    else:
+      fatal "File does not exist: " & args[0]
+  of "tag":
+    projectCmd()
+    if args.len == 0:
+      tag(c, ord(patch))
+    elif args[0].len == 1 and args[0][0] in {'a'..'z'}:
+      let field = ord(args[0][0]) - ord('a')
+      tag(c, field)
+    elif args[0].len == 1 and args[0][0] in {'A'..'Z'}:
+      let field = ord(args[0][0]) - ord('A')
+      tag(c, field)
+    elif '.' in args[0]:
+      tag(c, args[0])
+    else:
+      var field: SemVerField
+      try: field = parseEnum[SemVerField](args[0])
+      except: fatal "tag command takes one of 'patch' 'minor' 'major', a SemVer tag, or a letter from 'a' to 'z'"
+      tag(c, ord(field))
+  of "build", "test", "doc", "tasks":
+    projectCmd()
+    nimbleExec(action, args)
+  of "task":
+    projectCmd()
+    nimbleExec("", args)
+  of "env":
+    singleArg()
+    setupNimEnv c, args[0]
+  else:
+    fatal "Invalid action: " & action
+
+when isMainModule:
+  main()