fixes #19396; Nimdoc hide nonexported fields (#21305)

* suppresses non-exported fields of types and adds command-line option to re-enable this if desired

* corrected the doctest that produced a CI error

* an embarrassingly bad error in reasoning

* modified a nimdoc test to reflect updated behavior

* needed another change to bring utils.html doctest in sync with update

* add info

* fix nimdoc

* lint

* render postfix

* fixes a problem

* fixes nimdoc

* fix nimdoc

---------

Co-authored-by: johnperry-math <john.perry@usm.edu>
Co-authored-by: johnperry-math <devotus@yahoo.com>

13 files changed, 56 insertions, 24 deletions

diff --git a/compiler/commands.nim b/compiler/commands.nim
index 59b221647..7c0f7ab0a 100644
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -1008,6 +1008,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     expectNoArg(conf, switch, arg, pass, info)
     conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "shownonexports":
+    expectNoArg(conf, switch, arg, pass, info)
+    showNonExportedFields(conf)
   of "exceptions":
     case arg.normalize
     of "cpp": conf.exc = excCpp
diff --git a/compiler/docgen.nim b/compiler/docgen.nim
index cab334e7a..8105cc807 100644
--- a/compiler/docgen.nim
+++ b/compiler/docgen.nim
@@ -1008,7 +1008,7 @@ proc toLangSymbol(k: TSymKind, n: PNode, baseName: string): LangSymbol =
 
   if k == skType: result.symTypeKind = getTypeKind(n)
 
-proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
+proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags, nonExports: bool = false) =
   if (docFlags != kForceExport) and not isVisible(d, nameNode): return
   let
     name = getName(nameNode)
@@ -1062,8 +1062,11 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
                tooltip = detailedName, langSym = rstLangSymbol,
                priority = symbolPriority(k), info = lineinfo)
 
-  nodeToHighlightedHtml(d, n, result, {renderNoBody, renderNoComments,
-    renderDocComments, renderSyms, renderExpandUsing}, symbolOrIdEnc)
+  let renderFlags =
+    if nonExports: {renderNoBody, renderNoComments, renderDocComments, renderSyms,
+      renderExpandUsing, renderNonExportedFields}
+    else: {renderNoBody, renderNoComments, renderDocComments, renderSyms, renderExpandUsing}
+  nodeToHighlightedHtml(d, n, result, renderFlags, symbolOrIdEnc)
 
   let seeSrc = genSeeSrc(d, toFullPath(d.conf, n.info), n.info.line.int)
 
@@ -1311,12 +1314,13 @@ proc documentRaises*(cache: IdentCache; n: PNode) =
     if p5 != nil: n[pragmasPos].add p5
     if p6 != nil: n[pragmasPos].add p6
 
-proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
+proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags = kDefault) =
   ## Goes through nim nodes recursively and collects doc comments.
   ## Main function for `doc`:option: command,
   ## which is implemented in ``docgen2.nim``.
   template genItemAux(skind) =
     genItem(d, n, n[namePos], skind, docFlags)
+  let showNonExports = optShowNonExportedFields in config.globalOptions
   case n.kind
   of nkPragma:
     let pragmaNode = findPragma(n, wDeprecated)
@@ -1343,20 +1347,20 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
       if n[i].kind != nkCommentStmt:
         # order is always 'type var let const':
         genItem(d, n[i], n[i][0],
-                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags)
+                succ(skType, ord(n.kind)-ord(nkTypeSection)), docFlags, showNonExports)
   of nkStmtList:
-    for i in 0..<n.len: generateDoc(d, n[i], orig)
+    for i in 0..<n.len: generateDoc(d, n[i], orig, config)
   of nkWhenStmt:
     # generate documentation for the first branch only:
     if not checkForFalse(n[0][0]):
-      generateDoc(d, lastSon(n[0]), orig)
+      generateDoc(d, lastSon(n[0]), orig, config)
   of nkImportStmt:
     for it in n: traceDeps(d, it)
   of nkExportStmt:
     for it in n:
       if it.kind == nkSym:
         if d.module != nil and d.module == it.sym.owner:
-          generateDoc(d, it.sym.ast, orig, kForceExport)
+          generateDoc(d, it.sym.ast, orig, config, kForceExport)
         elif it.sym.ast != nil:
           exportSym(d, it.sym)
   of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
@@ -1786,7 +1790,7 @@ proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
   var ast = parseFile(conf.projectMainIdx, cache, conf)
   if ast == nil: return
   var d = newDocumentor(conf.projectFull, cache, conf, hasToc = true)
-  generateDoc(d, ast, ast)
+  generateDoc(d, ast, ast, conf)
   finishGenerateDoc(d)
   writeOutput(d)
   generateIndex(d)
diff --git a/compiler/docgen2.nim b/compiler/docgen2.nim
index ea3253a2c..e5669937f 100644
--- a/compiler/docgen2.nim
+++ b/compiler/docgen2.nim
@@ -50,7 +50,7 @@ proc processNode(c: PPassContext, n: PNode): PNode =
   result = n
   var g = PGen(c)
   if shouldProcess(g):
-    generateDoc(g.doc, n, n)
+    generateDoc(g.doc, n, n, g.config)
 
 proc processNodeJson(c: PPassContext, n: PNode): PNode =
   result = n
diff --git a/compiler/options.nim b/compiler/options.nim
index 8c477f0a5..833d86a16 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -108,6 +108,7 @@ type                          # please make sure we have under 32 options
     optSourcemap
     optProfileVM              # enable VM profiler
     optEnableDeepCopy         # ORC specific: enable 'deepcopy' for all types.
+    optShowNonExportedFields  # for documentation: show fields that are not exported
 
   TGlobalOptions* = set[TGlobalOption]
 
@@ -1004,6 +1005,9 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool =
   result = optDynlibOverrideAll in conf.globalOptions or
      conf.dllOverrides.hasKey(lib.canonDynlibName)
 
+proc showNonExportedFields*(conf: ConfigRef) =
+  incl(conf.globalOptions, optShowNonExportedFields)
+
 proc expandDone*(conf: ConfigRef): bool =
   result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress
 
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index 87a5e6580..4ed9ec177 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -23,7 +23,8 @@ type
   TRenderFlag* = enum
     renderNone, renderNoBody, renderNoComments, renderDocComments,
     renderNoPragmas, renderIds, renderNoProcDefs, renderSyms, renderRunnableExamples,
-    renderIr, renderExpandUsing
+    renderIr, renderNonExportedFields, renderExpandUsing
+
   TRenderFlags* = set[TRenderFlag]
   TRenderTok* = object
     kind*: TokType
@@ -652,7 +653,7 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
     inHideable = false
 
 proc gcomma(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
-            theEnd: int = - 1) =
+            theEnd: int = -1) =
   var ind: int
   if rfInConstExpr in c.flags:
     ind = g.indent + IndentWidth
@@ -1481,9 +1482,11 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) =
   of nkRecList:
     indentNL(g)
     for i in 0..<n.len:
-      optNL(g)
-      gsub(g, n[i], c)
-      gcoms(g)
+      if n[i].kind == nkIdentDefs and n[i][0].kind == nkPostfix or
+                        renderNonExportedFields in g.flags:
+        optNL(g)
+        gsub(g, n[i], c)
+        gcoms(g)
     dedent(g)
     putNL(g)
   of nkOfInherit:
diff --git a/compiler/vm.nim b/compiler/vm.nim
index 424685e40..3946be0fd 100644
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1603,7 +1603,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcRepr:
       decodeB(rkNode)
       createStr regs[ra]
-      regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments, renderDocComments})
+      regs[ra].node.strVal = renderTree(regs[rb].regToNode, {renderNoComments, renderDocComments, renderNonExportedFields})
     of opcQuit:
       if c.mode in {emRepl, emStaticExpr, emStaticStmt}:
         message(c.config, c.debug[pc], hintQuitCalled)
diff --git a/doc/docgen.md b/doc/docgen.md
index 70b41f17d..9e0a83bab 100644
--- a/doc/docgen.md
+++ b/doc/docgen.md
@@ -44,6 +44,7 @@ Generate HTML documentation for a whole project:
   # or `$nimcache/htmldocs` with `--usenimcache` which avoids clobbering your sources;
   # and likewise without `--project`.
   # Adding `-r` will open in a browser directly.
+  # Use `--showNonExports` to show non-exported fields of an exported type.
   ```
 
 Documentation Comments
diff --git a/nimdoc/extlinks/project/expected/main.html b/nimdoc/extlinks/project/expected/main.html
index 5facedb3b..cf7982fde 100644
--- a/nimdoc/extlinks/project/expected/main.html
+++ b/nimdoc/extlinks/project/expected/main.html
@@ -55,8 +55,7 @@
   <details open>
     <summary><a class="reference reference-toplevel" href="#7" id="57">Types</a></summary>
     <ul class="simple simple-toc-section">
-      <li><a class="reference" href="#A" title="A = object
-  x: int">A</a></li>
+      <li><a class="reference" href="#A" title="A = object">A</a></li>
 
     </ul>
   </details>
@@ -97,8 +96,7 @@
   <dl class="item">
     <div id="A">
   <dt><pre><a href="main.html#A"><span class="Identifier">A</span></a> <span class="Other">=</span> <span class="Keyword">object</span>
-  <span class="Identifier">x</span><span class="Other">:</span> <span class="Identifier">int</span>
-</pre></dt>
+  </pre></dt>
   <dd>
     
     
diff --git a/nimdoc/testproject/expected/subdir/subdir_b/utils.html b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
index f0f30536f..ee2618ab3 100644
--- a/nimdoc/testproject/expected/subdir/subdir_b/utils.html
+++ b/nimdoc/testproject/expected/subdir/subdir_b/utils.html
@@ -59,8 +59,7 @@
   <details open>
     <summary><a class="reference reference-toplevel" href="#7" id="57">Types</a></summary>
     <ul class="simple simple-toc-section">
-      <li><a class="reference" href="#G" title="G[T] = object
-  val: T">G</a></li>
+      <li><a class="reference" href="#G" title="G[T] = object">G</a></li>
 <li><a class="reference" href="#SomeType" title="SomeType = enum
   enumValueA, enumValueB, enumValueC">SomeType</a></li>
 
@@ -254,8 +253,7 @@ Ref. <a class="reference internal nimdoc" title="proc `[]`[T](x: G[T]): T" href=
   <dl class="item">
     <div id="G">
   <dt><pre><a href="utils.html#G"><span class="Identifier">G</span></a><span class="Other">[</span><span class="Identifier">T</span><span class="Other">]</span> <span class="Other">=</span> <span class="Keyword">object</span>
-  <span class="Identifier">val</span><span class="Other">:</span> <span class="Identifier">T</span>
-</pre></dt>
+  </pre></dt>
   <dd>
     
     
diff --git a/nimdoc/testproject/expected/testproject.html b/nimdoc/testproject/expected/testproject.html
index 7d10d7900..45a53bb93 100644
--- a/nimdoc/testproject/expected/testproject.html
+++ b/nimdoc/testproject/expected/testproject.html
@@ -65,6 +65,8 @@
   Circle,                   ## A circle
   Triangle,                 ## A three-sided shape
   Rectangle                  ## A four-sided shape">Shapes</a></li>
+<li><a class="reference" href="#T19396" title="T19396 = object
+  a*: int">T19396</a></li>
 
     </ul>
   </details>
@@ -401,6 +403,16 @@
     
   </dd>
 </div>
+<div id="T19396">
+  <dt><pre><a href="testproject.html#T19396"><span class="Identifier">T19396</span></a> <span class="Other">=</span> <span class="Keyword">object</span>
+  <span class="Identifier">a</span><span class="Operator">*</span><span class="Other">:</span> <span class="Identifier">int</span>
+</pre></dt>
+  <dd>
+    
+    
+    
+  </dd>
+</div>
 
   </dl>
 </div>
diff --git a/nimdoc/testproject/expected/testproject.idx b/nimdoc/testproject/expected/testproject.idx
index e69bedf33..46ffaee72 100644
--- a/nimdoc/testproject/expected/testproject.idx
+++ b/nimdoc/testproject/expected/testproject.idx
@@ -63,5 +63,6 @@ nim	Triangle	testproject.html#Triangle	Shapes.Triangle		380
 nim	Rectangle	testproject.html#Rectangle	Shapes.Rectangle		380
 nim	Shapes	testproject.html#Shapes	enum Shapes		380
 nim	anything	testproject.html#anything	proc anything()		387
+nim	T19396	testproject.html#T19396	object T19396		392
 nimgrp	bar	testproject.html#bar-procs-all	proc		31
 nimgrp	baz	testproject.html#baz-procs-all	proc		34
diff --git a/nimdoc/testproject/expected/theindex.html b/nimdoc/testproject/expected/theindex.html
index 7b4a95561..f9488eb2d 100644
--- a/nimdoc/testproject/expected/theindex.html
+++ b/nimdoc/testproject/expected/theindex.html
@@ -308,6 +308,10 @@
 <li><a class="reference external"
           data-doc-search-tag="testproject: var someVariable" href="testproject.html#someVariable">testproject: var someVariable</a></li>
           </ul></dd>
+<dt><a name="T19396" href="#T19396"><span>T19396:</span></a></dt><dd><ul class="simple">
+<li><a class="reference external"
+          data-doc-search-tag="testproject: object T19396" href="testproject.html#T19396">testproject: object T19396</a></li>
+          </ul></dd>
 <dt><a name="testNimDocTrailingExample" href="#testNimDocTrailingExample"><span>testNimDocTrailingExample:</span></a></dt><dd><ul class="simple">
 <li><a class="reference external"
           data-doc-search-tag="testproject: template testNimDocTrailingExample()" href="testproject.html#testNimDocTrailingExample.t">testproject: template testNimDocTrailingExample()</a></li>
diff --git a/nimdoc/testproject/testproject.nim b/nimdoc/testproject/testproject.nim
index d236552ac..b5fa2ac37 100644
--- a/nimdoc/testproject/testproject.nim
+++ b/nimdoc/testproject/testproject.nim
@@ -388,3 +388,7 @@ when true: # issue #15184
     ##
     ##  There is no block quote after blank lines at the beginning.
   discard
+
+type T19396* = object # bug #19396
+   a*: int
+   b: float