From 278c265c6fe1a3bc9f1fb3c4895d3108134b8294 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Wed, 6 Aug 2014 21:19:20 +0200 Subject: Parallelizes documentation building. --- tools/nimweb.nim | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/tools/nimweb.nim b/tools/nimweb.nim index 9a83a5cca..d3c6d3fcd 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -215,6 +215,11 @@ proc exec(cmd: string) = echo(cmd) if os.execShellCmd(cmd) != 0: quit("external program failed") +proc mexec(cmds: openarray[string]) = + ## Multiprocessor version of exec + if 0 != execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}): + quit("external program failed") + proc buildDocSamples(c: var TConfigData, destPath: string) = ## Special case documentation sample proc. ## @@ -229,18 +234,26 @@ proc buildDocSamples(c: var TConfigData, destPath: string) = proc buildDoc(c: var TConfigData, destPath: string) = # call nim for the documentation: + var + commands = newSeq[string](len(c.doc) + len(c.srcdoc) + len(c.srcdoc2)) + i = 0 for d in items(c.doc): - exec("nimrod rst2html $# --docSeeSrcUrl:$# -o:$# --index:on $#" % + commands[i] = "nimrod rst2html $# --docSeeSrcUrl:$# -o:$# --index:on $#" % [c.nimrodArgs, c.gitCommit, - destPath / changeFileExt(splitFile(d).name, "html"), d]) + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc for d in items(c.srcdoc): - exec("nimrod doc $# --docSeeSrcUrl:$# -o:$# --index:on $#" % + commands[i] = "nimrod doc $# --docSeeSrcUrl:$# -o:$# --index:on $#" % [c.nimrodArgs, c.gitCommit, - destPath / changeFileExt(splitFile(d).name, "html"), d]) + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc for d in items(c.srcdoc2): - exec("nimrod doc2 $# --docSeeSrcUrl:$# -o:$# --index:on $#" % + commands[i] = "nimrod doc2 $# --docSeeSrcUrl:$# -o:$# --index:on $#" % [c.nimrodArgs, c.gitCommit, - destPath / changeFileExt(splitFile(d).name, "html"), d]) + destPath / changeFileExt(splitFile(d).name, "html"), d] + i.inc + + mexec(commands) exec("nimrod buildIndex -o:$1/theindex.html $1" % [destPath]) proc buildPdfDoc(c: var TConfigData, destPath: string) = @@ -264,10 +277,12 @@ proc buildPdfDoc(c: var TConfigData, destPath: string) = proc buildAddDoc(c: var TConfigData, destPath: string) = # build additional documentation (without the index): - for d in items(c.webdoc): - exec("nimrod doc $# --docSeeSrcUrl:$# -o:$# $#" % + var commands = newSeq[string](c.webdoc.len) + for i, doc in pairs(c.webdoc): + commands[i] = "nimrod doc $# --docSeeSrcUrl:$# -o:$# $#" % [c.nimrodArgs, c.gitCommit, - destPath / changeFileExt(splitFile(d).name, "html"), d]) + destPath / changeFileExt(splitFile(doc).name, "html"), doc] + mexec(commands) proc parseNewsTitles(inputFilename: string): seq[TRssItem] = # parses the file for titles and returns them as TRssItem blocks. -- cgit 1.4.1-2-gfad0 From bab41babeafc69a14f757f55e9a876315089f1e7 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Fri, 8 Aug 2014 02:21:13 +0200 Subject: Adds --parallelBuild to koch web and automatic reruns. Refs #1455. --- doc/koch.txt | 7 +++++++ tools/nimweb.nim | 31 +++++++++++++++++++++++++++---- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/doc/koch.txt b/doc/koch.txt index 032151808..7be2be6d4 100644 --- a/doc/koch.txt +++ b/doc/koch.txt @@ -138,6 +138,13 @@ from rst to HTML. It also repeats the same operation but places the result in the ``web/upload`` which can be used to update the website at http://nimrod-lang.org. +By default the documentation will be built in parallel using the number of +available CPU cores. If any documentation build sub commands fail, they will +be rerun in serial fashion so that meaninful error output can be gathered for +inspection. The ``--parallelBuild:n`` switch or configuration option can be +used to force a specific number of parallel jobs or run everything serially +from the start (``n == 1``). + zip command ----------- diff --git a/tools/nimweb.nim b/tools/nimweb.nim index d3c6d3fcd..d76c5e354 100644 --- a/tools/nimweb.nim +++ b/tools/nimweb.nim @@ -21,6 +21,7 @@ type nimrodArgs: string gitCommit: string quotations: TTable[string, tuple[quote, author: string]] + numProcessors: int # Set by parallelBuild:n, only works for values > 0. TRssItem = object year, month, day, title: string @@ -42,6 +43,7 @@ proc initConfigData(c: var TConfigData) = c.ticker = "" c.vars = newStringTable(modeStyleInsensitive) c.gitCommit = "master" + c.numProcessors = countProcessors() # Attempts to obtain the git current commit. let (output, code) = execCmdEx("git log -n 1 --format=%H") if code == 0 and output.strip.len == 40: @@ -121,6 +123,12 @@ proc parseCmdLine(c: var TConfigData) = stdout.write(Version & "\n") quit(0) of "o", "output": c.outdir = val + of "parallelbuild": + try: + let num = parseInt(val) + if num != 0: c.numProcessors = num + except EInvalidValue: + quit("invalid numeric value for --parallelBuild") of "var": var idx = val.find('=') if idx < 0: quit("invalid command line") @@ -187,6 +195,12 @@ proc parseIniFile(c: var TConfigData) = of "srcdoc": addFiles(c.srcdoc, "lib", ".nim", split(v, {';'})) of "srcdoc2": addFiles(c.srcdoc2, "lib", ".nim", split(v, {';'})) of "webdoc": addFiles(c.webdoc, "lib", ".nim", split(v, {';'})) + of "parallelbuild": + try: + let num = parseInt(v) + if num != 0: c.numProcessors = num + except EInvalidValue: + quit("invalid numeric value for --parallelBuild in config") else: quit(errorStr(p, "unknown variable: " & k.key)) of "quotations": let vSplit = v.split('-') @@ -215,10 +229,19 @@ proc exec(cmd: string) = echo(cmd) if os.execShellCmd(cmd) != 0: quit("external program failed") -proc mexec(cmds: openarray[string]) = +proc sexec(cmds: openarray[string]) = + ## Serial queue wrapper around exec. + for cmd in cmds: exec(cmd) + +proc mexec(cmds: openarray[string], processors: int) = ## Multiprocessor version of exec + if processors < 2: + sexec(cmds) + return + if 0 != execProcesses(cmds, {poStdErrToStdOut, poParentStreams, poEchoCmd}): - quit("external program failed") + echo "external program failed, retrying serial work queue for logs!" + sexec(cmds) proc buildDocSamples(c: var TConfigData, destPath: string) = ## Special case documentation sample proc. @@ -253,7 +276,7 @@ proc buildDoc(c: var TConfigData, destPath: string) = destPath / changeFileExt(splitFile(d).name, "html"), d] i.inc - mexec(commands) + mexec(commands, c.numProcessors) exec("nimrod buildIndex -o:$1/theindex.html $1" % [destPath]) proc buildPdfDoc(c: var TConfigData, destPath: string) = @@ -282,7 +305,7 @@ proc buildAddDoc(c: var TConfigData, destPath: string) = commands[i] = "nimrod doc $# --docSeeSrcUrl:$# -o:$# $#" % [c.nimrodArgs, c.gitCommit, destPath / changeFileExt(splitFile(doc).name, "html"), doc] - mexec(commands) + mexec(commands, c.numProcessors) proc parseNewsTitles(inputFilename: string): seq[TRssItem] = # parses the file for titles and returns them as TRssItem blocks. -- cgit 1.4.1-2-gfad0