diff options
Diffstat (limited to 'compiler/ic/rodfiles.nim')
| -rw-r--r-- | compiler/ic/rodfiles.nim | 130 |
1 files changed, 119 insertions, 11 deletions
diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim index a518870f8..ac995dd2e 100644 --- a/compiler/ic/rodfiles.nim +++ b/compiler/ic/rodfiles.nim @@ -7,7 +7,68 @@ # distribution, for details about the copyright. # -from typetraits import supportsCopyMem +## Low level binary format used by the compiler to store and load various AST +## and related data. +## +## NB: this is incredibly low level and if you're interested in how the +## compiler works and less a storage format, you're probably looking for +## the `ic` or `packed_ast` modules to understand the logical format. + +from std/typetraits import supportsCopyMem + +when defined(nimPreviewSlimSystem): + import std/[syncio, assertions] + +import std / tables + +## Overview +## ======== +## `RodFile` represents a Rod File (versioned binary format), and the +## associated data for common interactions such as IO and error tracking +## (`RodFileError`). The file format broken up into sections (`RodSection`) +## and preceded by a header (see: `cookie`). The precise layout, section +## ordering and data following the section are determined by the user. See +## `ic.loadRodFile`. +## +## A basic but "wrong" example of the lifecycle: +## --------------------------------------------- +## 1. `create` or `open` - create a new one or open an existing +## 2. `storeHeader` - header info +## 3. `storePrim` or `storeSeq` - save your stuff +## 4. `close` - and we're done +## +## Now read the bits below to understand what's missing. +## +## ### Issues with the Example +## Missing Sections: +## This is a low level API, so headers and sections need to be stored and +## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & +## `loadSection`, respectively. +## +## No Error Handling: +## The API is centered around IO and prone to error, each operation checks or +## sets the `RodFile.err` field. A user of this API needs to handle these +## appropriately. +## +## API Notes +## ========= +## +## Valid inputs for Rod files +## -------------------------- +## ASTs, hopes, dreams, and anything as long as it and any children it may have +## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: +## * string +## * objects & tuples (fields are recursed) +## * sequences AKA `seq[T]` +## +## Note on error handling style +## ---------------------------- +## A flag based approach is used where operations no-op in case of a +## preexisting error and set the flag if they encounter one. +## +## Misc +## ---- +## * 'Prim' is short for 'primitive', as in a non-sequence type type RodSection* = enum @@ -16,16 +77,15 @@ type stringsSection checkSumsSection depsSection - integersSection - floatsSection + numbersSection exportsSection + hiddenSection reexportsSection compilerProcsSection trmacrosSection convertersSection methodsSection pureEnumsSection - macroUsagesSection toReplaySection topLevelSection bodiesSection @@ -34,10 +94,16 @@ type typeInstCacheSection procInstCacheSection attachedOpsSection - methodsPerTypeSection + methodsPerGenericTypeSection enumToStringProcsSection + methodsPerTypeSection + dispatchersSection typeInfoSection # required by the backend + backendFlagsSection aliveSymsSection # beware, this is stored in a `.alivesyms` file. + sideChannelSection + namespaceSection + symnamesSection RodFileError* = enum ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch, @@ -50,8 +116,8 @@ type # better than exceptions. const - RodVersion = 1 - cookie = [byte(0), byte('R'), byte('O'), byte('D'), + RodVersion = 2 + defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'), byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] proc setError(f: var RodFile; err: RodFileError) {.inline.} = @@ -59,6 +125,8 @@ proc setError(f: var RodFile; err: RodFileError) {.inline.} = #raise newException(IOError, "IO error") proc storePrim*(f: var RodFile; s: string) = + ## Stores a string. + ## The len is prefixed to allow for later retreival. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -72,6 +140,9 @@ proc storePrim*(f: var RodFile; s: string) = setError f, ioFailure proc storePrim*[T](f: var RodFile; x: T) = + ## Stores a non-sequence/string `T`. + ## If `T` doesn't support `copyMem` and is an object or tuple then the fields + ## are written -- the user from context will need to know which `T` to load. if f.err != ok: return when supportsCopyMem(T): if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -89,6 +160,7 @@ proc storePrim*[T](f: var RodFile; x: T) = {.error: "unsupported type for 'storePrim'".} proc storeSeq*[T](f: var RodFile; s: seq[T]) = + ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. if f.err != ok: return if s.len >= high(int32): setError f, tooBig @@ -100,7 +172,20 @@ proc storeSeq*[T](f: var RodFile; s: seq[T]) = for i in 0..<s.len: storePrim(f, s[i]) +proc storeOrderedTable*[K, T](f: var RodFile; s: OrderedTable[K, T]) = + if f.err != ok: return + if s.len >= high(int32): + setError f, tooBig + return + var lenPrefix = int32(s.len) + if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + setError f, ioFailure + else: + for _, v in s: + storePrim(f, v) + proc loadPrim*(f: var RodFile; s: var string) = + ## Read a string, the length was stored as a prefix if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -112,6 +197,7 @@ proc loadPrim*(f: var RodFile; s: var string) = setError f, ioFailure proc loadPrim*[T](f: var RodFile; x: var T) = + ## Load a non-sequence/string `T`. if f.err != ok: return when supportsCopyMem(T): if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): @@ -129,6 +215,7 @@ proc loadPrim*[T](f: var RodFile; x: var T) = {.error: "unsupported type for 'loadPrim'".} proc loadSeq*[T](f: var RodFile; s: var seq[T]) = + ## `T` must be compatible with `copyMem`, see `loadPrim` if f.err != ok: return var lenPrefix = int32(0) if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): @@ -138,38 +225,59 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) = for i in 0..<lenPrefix: loadPrim(f, s[i]) -proc storeHeader*(f: var RodFile) = +proc loadOrderedTable*[K, T](f: var RodFile; s: var OrderedTable[K, T]) = + ## `T` must be compatible with `copyMem`, see `loadPrim` + if f.err != ok: return + var lenPrefix = int32(0) + if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): + setError f, ioFailure + else: + s = initOrderedTable[K, T](lenPrefix) + for i in 0..<lenPrefix: + var x = default T + loadPrim(f, x) + s[x.id] = x + +proc storeHeader*(f: var RodFile; cookie = defaultCookie) = + ## stores the header which is described by `cookie`. if f.err != ok: return if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len: setError f, ioFailure -proc loadHeader*(f: var RodFile) = +proc loadHeader*(f: var RodFile; cookie = defaultCookie) = + ## Loads the header which is described by `cookie`. if f.err != ok: return - var thisCookie: array[cookie.len, byte] + var thisCookie: array[cookie.len, byte] = default(array[cookie.len, byte]) if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len: setError f, ioFailure elif thisCookie != cookie: setError f, wrongHeader proc storeSection*(f: var RodFile; s: RodSection) = + ## update `currentSection` and writes the bytes value of s. if f.err != ok: return assert f.currentSection < s f.currentSection = s storePrim(f, s) proc loadSection*(f: var RodFile; expected: RodSection) = + ## read the bytes value of s, sets and error if the section is incorrect. if f.err != ok: return - var s: RodSection + var s: RodSection = default(RodSection) loadPrim(f, s) if expected != s and f.err == ok: setError f, wrongSection proc create*(filename: string): RodFile = + ## create the file and open it for writing + result = default(RodFile) if not open(result.f, filename, fmWrite): setError result, cannotOpen proc close*(f: var RodFile) = close(f.f) proc open*(filename: string): RodFile = + ## open the file for reading + result = default(RodFile) if not open(result.f, filename, fmRead): setError result, cannotOpen |