[ci skip] document compiler/ic/rodfiles.nim (#17771)

* [ci skip] document compiler/ic/rodfiles.nim

Why?
* understand how rodfile module works and a bit of the format
* leave notes behind for others
* rather than Araq guess what others need, he can fix what other glean
* possible model for making the compiler more aproachable

Bonus:
* might have found a minor bug in `loadSection`

* Update compiler/ic/rodfiles.nim

Co-authored-by: Andreas Rumpf <rumpf_a@web.de>

1 files changed, 72 insertions, 0 deletions

diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim
index 47362da0b..86d8ba203 100644
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -7,8 +7,65 @@
 #    distribution, for details about the copyright.
 #
 
+## Low level binary format used by the compiler to store and load various AST
+## and related data.
+##
+## NB: this is incredibly low level and if you're interested in how the
+##     compiler works and less a storage format, you're probably looking for
+##     the `ic` or `packed_ast` modules to understand the logical format.
+
 from typetraits import supportsCopyMem
 
+## Overview
+## ========
+## `RodFile` represents a Rod File (versioned binary format), and the
+## associated data for common interactions such as IO and error tracking
+## (`RodFileError`). The file format broken up into sections (`RodSection`)
+## and preceeded by a header (see: `cookie`). The precise layout, section
+## ordering and data following the section are determined by the user. See
+## `ic.loadRoadFile`.
+## 
+## A basic but "wrong" example of the lifecycle:
+## ---------------------------------------------
+## 1. `create` or `open`        - create a new one or open an existing
+## 2. `storeHeader`             - header info
+## 3. `storePrim` or `storeSeq` - save your stuff
+## 4. `close`                   - and we're done
+##
+## Now read the bits below to understand what's missing.
+##
+## Issues with the Example
+## ```````````````````````
+## Missing Sections:
+## This is a low level API, so headers and sections need to be stored and
+## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &
+## `loadSection`, respectively.
+##
+## No Error Handling:
+## The API is centered around IO and prone to error, each operation checks or
+## sets the `RodFile.err` field. A user of this API needs to handle these
+## appropriately.
+## 
+## API Notes
+## =========
+## 
+## Valid inputs for Rod files
+## --------------------------
+## ASTs, hopes, dreams, and anything as long as it and any children it may have
+## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are:
+## * string
+## * objects & tuples (fields are recursed)
+## * sequences AKA `seq[T]`
+##
+## Note on error handling style
+## ----------------------------
+## A flag based approach is used where operations no-op in case of a
+## preexisting error and set the flag if they encounter one.
+## 
+## Misc
+## ----
+## * 'Prim' is short for 'primitive', as in a non-sequence type
+
 type
   RodSection* = enum
     versionSection
@@ -60,6 +117,8 @@ proc setError(f: var RodFile; err: RodFileError) {.inline.} =
   #raise newException(IOError, "IO error")
 
 proc storePrim*(f: var RodFile; s: string) =
+  ## Stores a string.
+  ## The len is prefixed to allow for later retreival.
   if f.err != ok: return
   if s.len >= high(int32):
     setError f, tooBig
@@ -73,6 +132,9 @@ proc storePrim*(f: var RodFile; s: string) =
         setError f, ioFailure
 
 proc storePrim*[T](f: var RodFile; x: T) =
+  ## Stores a non-sequence/string `T`.
+  ## If `T` doesn't support `copyMem` and is an object or tuple then the fields
+  ## are written -- the user from context will need to know which `T` to load.
   if f.err != ok: return
   when supportsCopyMem(T):
     if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
@@ -90,6 +152,7 @@ proc storePrim*[T](f: var RodFile; x: T) =
     {.error: "unsupported type for 'storePrim'".}
 
 proc storeSeq*[T](f: var RodFile; s: seq[T]) =
+  ## Stores a sequence of `T`s, with the len as a prefix for later retrieval.
   if f.err != ok: return
   if s.len >= high(int32):
     setError f, tooBig
@@ -102,6 +165,7 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) =
       storePrim(f, s[i])
 
 proc loadPrim*(f: var RodFile; s: var string) =
+  ## Read a string, the length was stored as a prefix
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
@@ -113,6 +177,7 @@ proc loadPrim*(f: var RodFile; s: var string) =
         setError f, ioFailure
 
 proc loadPrim*[T](f: var RodFile; x: var T) =
+  ## Load a non-sequence/string `T`.
   if f.err != ok: return
   when supportsCopyMem(T):
     if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
@@ -130,6 +195,7 @@ proc loadPrim*[T](f: var RodFile; x: var T) =
     {.error: "unsupported type for 'loadPrim'".}
 
 proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
+  ## `T` must be compatible with `copyMem`, see `loadPrim`
   if f.err != ok: return
   var lenPrefix = int32(0)
   if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
@@ -140,11 +206,13 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
       loadPrim(f, s[i])
 
 proc storeHeader*(f: var RodFile) =
+  ## stores the header which is described by `cookie`.
   if f.err != ok: return
   if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
     setError f, ioFailure
 
 proc loadHeader*(f: var RodFile) =
+  ## Loads the header which is described by `cookie`.
   if f.err != ok: return
   var thisCookie: array[cookie.len, byte]
   if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
@@ -153,12 +221,14 @@ proc loadHeader*(f: var RodFile) =
     setError f, wrongHeader
 
 proc storeSection*(f: var RodFile; s: RodSection) =
+  ## update `currentSection` and writes the bytes value of s.
   if f.err != ok: return
   assert f.currentSection < s
   f.currentSection = s
   storePrim(f, s)
 
 proc loadSection*(f: var RodFile; expected: RodSection) =
+  ## read the bytes value of s, sets and error if the section is incorrect.
   if f.err != ok: return
   var s: RodSection
   loadPrim(f, s)
@@ -166,11 +236,13 @@ proc loadSection*(f: var RodFile; expected: RodSection) =
     setError f, wrongSection
 
 proc create*(filename: string): RodFile =
+  ## create the file and open it for writing
   if not open(result.f, filename, fmWrite):
     setError result, cannotOpen
 
 proc close*(f: var RodFile) = close(f.f)
 
 proc open*(filename: string): RodFile =
+  ## open the file for reading
   if not open(result.f, filename, fmRead):
     setError result, cannotOpen